2024-12-09 13:22:19,461 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-12-09 13:22:19,488 main DEBUG Took 0.025103 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 13:22:19,493 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 13:22:19,493 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 13:22:19,495 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 13:22:19,497 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 13:22:19,512 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 13:22:19,541 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 13:22:19,546 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 13:22:19,547 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 13:22:19,548 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 13:22:19,552 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 13:22:19,553 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 13:22:19,554 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 13:22:19,554 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 13:22:19,555 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 13:22:19,555 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 13:22:19,556 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 13:22:19,557 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 13:22:19,557 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 13:22:19,558 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 13:22:19,560 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 13:22:19,560 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 13:22:19,562 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 13:22:19,562 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 13:22:19,563 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 13:22:19,564 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 13:22:19,565 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 13:22:19,566 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 13:22:19,567 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 13:22:19,567 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 13:22:19,568 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 13:22:19,568 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 13:22:19,570 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 13:22:19,572 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 13:22:19,574 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 13:22:19,575 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 13:22:19,576 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 13:22:19,576 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 13:22:19,589 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 13:22:19,592 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 13:22:19,599 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 13:22:19,599 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 13:22:19,600 main DEBUG createAppenders(={Console}) 2024-12-09 13:22:19,600 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f initialized 2024-12-09 13:22:19,601 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-12-09 13:22:19,601 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f OK. 2024-12-09 13:22:19,602 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 13:22:19,602 main DEBUG OutputStream closed 2024-12-09 13:22:19,603 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 13:22:19,604 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 13:22:19,605 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@61001b64 OK 2024-12-09 13:22:19,738 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 13:22:19,741 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 13:22:19,743 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 13:22:19,745 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 13:22:19,745 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 13:22:19,746 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 13:22:19,746 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 13:22:19,747 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 13:22:19,747 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 13:22:19,748 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 13:22:19,748 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 13:22:19,748 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 13:22:19,749 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 13:22:19,749 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 13:22:19,750 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 13:22:19,750 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 13:22:19,751 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 13:22:19,751 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 13:22:19,754 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 13:22:19,755 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4ed5eb72) with optional ClassLoader: null 2024-12-09 13:22:19,756 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 13:22:19,757 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4ed5eb72] started OK. 2024-12-09T13:22:20,191 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d 2024-12-09 13:22:20,195 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 13:22:20,196 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T13:22:20,213 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-09T13:22:20,258 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-09T13:22:20,339 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-09T13:22:20,339 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-09T13:22:20,356 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-09T13:22:20,378 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/cluster_b6088222-ea39-2e85-344a-dadc86871089, deleteOnExit=true 2024-12-09T13:22:20,378 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-09T13:22:20,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/test.cache.data in system properties and HBase conf 2024-12-09T13:22:20,380 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/hadoop.tmp.dir in system properties and HBase conf 2024-12-09T13:22:20,381 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/hadoop.log.dir in system properties and HBase conf 2024-12-09T13:22:20,381 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-09T13:22:20,382 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-09T13:22:20,382 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-09T13:22:20,517 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T13:22:20,680 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T13:22:20,687 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-09T13:22:20,688 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-09T13:22:20,688 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-09T13:22:20,689 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T13:22:20,689 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-09T13:22:20,690 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-09T13:22:20,690 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T13:22:20,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T13:22:20,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-09T13:22:20,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/nfs.dump.dir in system properties and HBase conf 2024-12-09T13:22:20,695 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/java.io.tmpdir in system properties and HBase conf 2024-12-09T13:22:20,695 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T13:22:20,696 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-09T13:22:20,696 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-09T13:22:21,913 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-09T13:22:22,018 INFO [Time-limited test {}] log.Log(170): Logging initialized @3627ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-09T13:22:22,124 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T13:22:22,222 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T13:22:22,267 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T13:22:22,267 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T13:22:22,269 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T13:22:22,287 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T13:22:22,291 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b5fa2d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/hadoop.log.dir/,AVAILABLE} 2024-12-09T13:22:22,293 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d655053{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T13:22:22,647 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7fe9bfd3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/java.io.tmpdir/jetty-localhost-39027-hadoop-hdfs-3_4_1-tests_jar-_-any-202457583769672166/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T13:22:22,668 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@16695a38{HTTP/1.1, (http/1.1)}{localhost:39027} 2024-12-09T13:22:22,668 INFO [Time-limited test {}] server.Server(415): Started @4277ms 2024-12-09T13:22:23,305 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T13:22:23,318 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T13:22:23,324 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T13:22:23,324 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T13:22:23,325 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T13:22:23,326 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1acb58f2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/hadoop.log.dir/,AVAILABLE} 2024-12-09T13:22:23,327 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@f666beb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T13:22:23,555 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3683c385{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/java.io.tmpdir/jetty-localhost-40175-hadoop-hdfs-3_4_1-tests_jar-_-any-9279159534062684995/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T13:22:23,559 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@e143a86{HTTP/1.1, (http/1.1)}{localhost:40175} 2024-12-09T13:22:23,559 INFO [Time-limited test {}] server.Server(415): Started @5168ms 2024-12-09T13:22:23,656 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T13:22:23,927 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T13:22:23,938 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T13:22:23,955 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T13:22:23,955 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T13:22:23,955 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-09T13:22:23,960 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1c37dc51{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/hadoop.log.dir/,AVAILABLE} 2024-12-09T13:22:23,961 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@17f0ed67{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T13:22:24,188 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@d332c17{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/java.io.tmpdir/jetty-localhost-34069-hadoop-hdfs-3_4_1-tests_jar-_-any-8530159463573238732/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T13:22:24,189 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1d2ea2b6{HTTP/1.1, (http/1.1)}{localhost:34069} 2024-12-09T13:22:24,190 INFO [Time-limited test {}] server.Server(415): Started @5799ms 2024-12-09T13:22:24,193 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T13:22:24,351 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T13:22:24,364 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T13:22:24,396 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T13:22:24,396 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T13:22:24,397 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T13:22:24,408 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@51314a16{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/hadoop.log.dir/,AVAILABLE} 2024-12-09T13:22:24,409 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@137df924{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T13:22:24,602 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/cluster_b6088222-ea39-2e85-344a-dadc86871089/data/data3/current/BP-1859740653-172.17.0.3-1733750541580/current, will proceed with Du for space computation calculation, 2024-12-09T13:22:24,607 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/cluster_b6088222-ea39-2e85-344a-dadc86871089/data/data4/current/BP-1859740653-172.17.0.3-1733750541580/current, will proceed with Du for space computation calculation, 2024-12-09T13:22:24,615 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/cluster_b6088222-ea39-2e85-344a-dadc86871089/data/data2/current/BP-1859740653-172.17.0.3-1733750541580/current, will proceed with Du for space computation calculation, 2024-12-09T13:22:24,618 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/cluster_b6088222-ea39-2e85-344a-dadc86871089/data/data1/current/BP-1859740653-172.17.0.3-1733750541580/current, will proceed with Du for space computation calculation, 2024-12-09T13:22:24,654 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@49d6f087{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/java.io.tmpdir/jetty-localhost-46689-hadoop-hdfs-3_4_1-tests_jar-_-any-9367875242261654559/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T13:22:24,657 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@245e172d{HTTP/1.1, (http/1.1)}{localhost:46689} 2024-12-09T13:22:24,657 INFO [Time-limited test {}] server.Server(415): Started @6267ms 2024-12-09T13:22:24,660 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T13:22:24,950 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T13:22:24,951 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T13:22:25,120 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa44e2ef0d4cb6b48 with lease ID 0x662478f1da61b497: Processing first storage report for DS-8af8162c-427e-4d04-be85-a94deb21afc3 from datanode DatanodeRegistration(127.0.0.1:37811, datanodeUuid=2cd4bdd8-71e6-4810-a8b4-c0ced52b755f, infoPort=34135, infoSecurePort=0, ipcPort=43635, storageInfo=lv=-57;cid=testClusterID;nsid=1859694849;c=1733750541580) 2024-12-09T13:22:25,121 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa44e2ef0d4cb6b48 with lease ID 0x662478f1da61b497: from storage DS-8af8162c-427e-4d04-be85-a94deb21afc3 node DatanodeRegistration(127.0.0.1:37811, datanodeUuid=2cd4bdd8-71e6-4810-a8b4-c0ced52b755f, infoPort=34135, infoSecurePort=0, ipcPort=43635, storageInfo=lv=-57;cid=testClusterID;nsid=1859694849;c=1733750541580), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-09T13:22:25,122 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe9824cf2b9052416 with lease ID 0x662478f1da61b496: Processing first storage report for DS-693cde34-0a47-46a5-8ae7-9edd9492bc06 from datanode DatanodeRegistration(127.0.0.1:41523, datanodeUuid=fce0c195-64e1-4f70-bc11-09e129f90579, infoPort=44161, infoSecurePort=0, ipcPort=34651, storageInfo=lv=-57;cid=testClusterID;nsid=1859694849;c=1733750541580) 2024-12-09T13:22:25,122 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe9824cf2b9052416 with lease ID 0x662478f1da61b496: from storage DS-693cde34-0a47-46a5-8ae7-9edd9492bc06 node DatanodeRegistration(127.0.0.1:41523, datanodeUuid=fce0c195-64e1-4f70-bc11-09e129f90579, infoPort=44161, infoSecurePort=0, ipcPort=34651, storageInfo=lv=-57;cid=testClusterID;nsid=1859694849;c=1733750541580), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T13:22:25,123 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe9824cf2b9052416 with lease ID 0x662478f1da61b496: Processing first storage report for DS-bc0017c8-3a43-4d04-946b-7d77ffb12511 from datanode DatanodeRegistration(127.0.0.1:41523, datanodeUuid=fce0c195-64e1-4f70-bc11-09e129f90579, infoPort=44161, infoSecurePort=0, ipcPort=34651, storageInfo=lv=-57;cid=testClusterID;nsid=1859694849;c=1733750541580) 2024-12-09T13:22:25,123 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe9824cf2b9052416 with lease ID 0x662478f1da61b496: from storage DS-bc0017c8-3a43-4d04-946b-7d77ffb12511 node DatanodeRegistration(127.0.0.1:41523, datanodeUuid=fce0c195-64e1-4f70-bc11-09e129f90579, infoPort=44161, infoSecurePort=0, ipcPort=34651, storageInfo=lv=-57;cid=testClusterID;nsid=1859694849;c=1733750541580), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T13:22:25,123 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa44e2ef0d4cb6b48 with lease ID 0x662478f1da61b497: Processing first storage report for DS-1b4dd7d8-a968-456b-bd51-d84a79edefa6 from datanode DatanodeRegistration(127.0.0.1:37811, datanodeUuid=2cd4bdd8-71e6-4810-a8b4-c0ced52b755f, infoPort=34135, infoSecurePort=0, ipcPort=43635, storageInfo=lv=-57;cid=testClusterID;nsid=1859694849;c=1733750541580) 2024-12-09T13:22:25,123 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa44e2ef0d4cb6b48 with lease ID 0x662478f1da61b497: from storage DS-1b4dd7d8-a968-456b-bd51-d84a79edefa6 node DatanodeRegistration(127.0.0.1:37811, datanodeUuid=2cd4bdd8-71e6-4810-a8b4-c0ced52b755f, infoPort=34135, infoSecurePort=0, ipcPort=43635, storageInfo=lv=-57;cid=testClusterID;nsid=1859694849;c=1733750541580), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T13:22:25,174 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/cluster_b6088222-ea39-2e85-344a-dadc86871089/data/data5/current/BP-1859740653-172.17.0.3-1733750541580/current, will proceed with Du for space computation calculation, 2024-12-09T13:22:25,175 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/cluster_b6088222-ea39-2e85-344a-dadc86871089/data/data6/current/BP-1859740653-172.17.0.3-1733750541580/current, will proceed with Du for space computation calculation, 2024-12-09T13:22:25,255 WARN [Thread-123 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T13:22:25,269 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5eac223076689828 with lease ID 0x662478f1da61b498: Processing first storage report for DS-bb1bc572-231a-4487-9b7d-89ab2125a50b from datanode DatanodeRegistration(127.0.0.1:40963, datanodeUuid=51e845fd-b4eb-4769-b479-94aa58893f5e, infoPort=34323, infoSecurePort=0, ipcPort=44493, storageInfo=lv=-57;cid=testClusterID;nsid=1859694849;c=1733750541580) 2024-12-09T13:22:25,269 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5eac223076689828 with lease ID 0x662478f1da61b498: from storage DS-bb1bc572-231a-4487-9b7d-89ab2125a50b node DatanodeRegistration(127.0.0.1:40963, datanodeUuid=51e845fd-b4eb-4769-b479-94aa58893f5e, infoPort=34323, infoSecurePort=0, ipcPort=44493, storageInfo=lv=-57;cid=testClusterID;nsid=1859694849;c=1733750541580), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T13:22:25,270 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5eac223076689828 with lease ID 0x662478f1da61b498: Processing first storage report for DS-bbd9d322-e3f3-4ec3-acf0-08ecb435df32 from datanode DatanodeRegistration(127.0.0.1:40963, datanodeUuid=51e845fd-b4eb-4769-b479-94aa58893f5e, infoPort=34323, infoSecurePort=0, ipcPort=44493, storageInfo=lv=-57;cid=testClusterID;nsid=1859694849;c=1733750541580) 2024-12-09T13:22:25,270 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5eac223076689828 with lease ID 0x662478f1da61b498: from storage DS-bbd9d322-e3f3-4ec3-acf0-08ecb435df32 node DatanodeRegistration(127.0.0.1:40963, datanodeUuid=51e845fd-b4eb-4769-b479-94aa58893f5e, infoPort=34323, infoSecurePort=0, ipcPort=44493, storageInfo=lv=-57;cid=testClusterID;nsid=1859694849;c=1733750541580), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T13:22:25,675 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d 2024-12-09T13:22:25,842 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/cluster_b6088222-ea39-2e85-344a-dadc86871089/zookeeper_0, clientPort=53371, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/cluster_b6088222-ea39-2e85-344a-dadc86871089/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/cluster_b6088222-ea39-2e85-344a-dadc86871089/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T13:22:25,877 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53371 2024-12-09T13:22:25,900 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T13:22:25,908 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T13:22:26,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741825_1001 (size=7) 2024-12-09T13:22:26,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741825_1001 (size=7) 2024-12-09T13:22:26,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741825_1001 (size=7) 2024-12-09T13:22:26,677 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1 with version=8 2024-12-09T13:22:26,678 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/hbase-staging 2024-12-09T13:22:27,073 INFO [Time-limited test {}] client.ConnectionUtils(128): master/cad571b91d50:0 server-side Connection retries=45 2024-12-09T13:22:27,088 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T13:22:27,089 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T13:22:27,095 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T13:22:27,096 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T13:22:27,096 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T13:22:27,391 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-09T13:22:27,507 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T13:22:27,520 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T13:22:27,526 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T13:22:27,565 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 30089 (auto-detected) 2024-12-09T13:22:27,566 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-09T13:22:27,594 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44823 2024-12-09T13:22:27,624 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44823 connecting to ZooKeeper ensemble=127.0.0.1:53371 2024-12-09T13:22:27,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:448230x0, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T13:22:27,729 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44823-0x1012b596d9a0000 connected 2024-12-09T13:22:27,894 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T13:22:27,898 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T13:22:27,922 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T13:22:27,935 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1, hbase.cluster.distributed=false 2024-12-09T13:22:28,010 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T13:22:28,020 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44823 2024-12-09T13:22:28,027 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44823 2024-12-09T13:22:28,034 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44823 2024-12-09T13:22:28,046 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44823 2024-12-09T13:22:28,048 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44823 2024-12-09T13:22:28,215 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/cad571b91d50:0 server-side Connection retries=45 2024-12-09T13:22:28,218 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T13:22:28,218 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T13:22:28,218 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T13:22:28,219 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T13:22:28,219 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T13:22:28,222 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T13:22:28,227 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T13:22:28,230 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39675 2024-12-09T13:22:28,233 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39675 connecting to ZooKeeper ensemble=127.0.0.1:53371 2024-12-09T13:22:28,236 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T13:22:28,242 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T13:22:28,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:396750x0, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T13:22:28,281 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39675-0x1012b596d9a0001 connected 2024-12-09T13:22:28,291 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39675-0x1012b596d9a0001, quorum=127.0.0.1:53371, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T13:22:28,298 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T13:22:28,314 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T13:22:28,322 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39675-0x1012b596d9a0001, quorum=127.0.0.1:53371, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T13:22:28,342 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39675-0x1012b596d9a0001, quorum=127.0.0.1:53371, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T13:22:28,344 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39675 2024-12-09T13:22:28,345 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39675 2024-12-09T13:22:28,349 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39675 2024-12-09T13:22:28,352 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39675 2024-12-09T13:22:28,353 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39675 2024-12-09T13:22:28,377 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/cad571b91d50:0 server-side Connection retries=45 2024-12-09T13:22:28,377 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T13:22:28,378 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T13:22:28,378 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T13:22:28,378 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T13:22:28,379 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T13:22:28,379 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T13:22:28,379 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T13:22:28,380 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34387 2024-12-09T13:22:28,383 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34387 connecting to ZooKeeper ensemble=127.0.0.1:53371 2024-12-09T13:22:28,384 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T13:22:28,388 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T13:22:28,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:343870x0, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T13:22:28,401 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:343870x0, quorum=127.0.0.1:53371, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T13:22:28,401 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T13:22:28,403 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34387-0x1012b596d9a0002 connected 2024-12-09T13:22:28,410 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T13:22:28,415 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34387-0x1012b596d9a0002, quorum=127.0.0.1:53371, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T13:22:28,418 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34387-0x1012b596d9a0002, quorum=127.0.0.1:53371, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T13:22:28,426 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34387 2024-12-09T13:22:28,428 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34387 2024-12-09T13:22:28,429 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34387 2024-12-09T13:22:28,430 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34387 2024-12-09T13:22:28,431 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34387 2024-12-09T13:22:28,454 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/cad571b91d50:0 server-side Connection retries=45 2024-12-09T13:22:28,454 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T13:22:28,454 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T13:22:28,455 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T13:22:28,455 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T13:22:28,455 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T13:22:28,455 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T13:22:28,456 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T13:22:28,457 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34317 2024-12-09T13:22:28,458 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34317 connecting to ZooKeeper ensemble=127.0.0.1:53371 2024-12-09T13:22:28,459 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T13:22:28,463 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T13:22:28,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:343170x0, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T13:22:28,470 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:343170x0, quorum=127.0.0.1:53371, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T13:22:28,470 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34317-0x1012b596d9a0003 connected 2024-12-09T13:22:28,471 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T13:22:28,474 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T13:22:28,475 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34317-0x1012b596d9a0003, quorum=127.0.0.1:53371, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T13:22:28,478 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34317-0x1012b596d9a0003, quorum=127.0.0.1:53371, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T13:22:28,479 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34317 2024-12-09T13:22:28,479 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34317 2024-12-09T13:22:28,480 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34317 2024-12-09T13:22:28,480 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34317 2024-12-09T13:22:28,481 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34317 2024-12-09T13:22:28,503 DEBUG [M:0;cad571b91d50:44823 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;cad571b91d50:44823 2024-12-09T13:22:28,504 INFO [master/cad571b91d50:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/cad571b91d50,44823,1733750546819 2024-12-09T13:22:28,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34317-0x1012b596d9a0003, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T13:22:28,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34387-0x1012b596d9a0002, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T13:22:28,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39675-0x1012b596d9a0001, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T13:22:28,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T13:22:28,525 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/cad571b91d50,44823,1733750546819 2024-12-09T13:22:28,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:22:28,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34317-0x1012b596d9a0003, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T13:22:28,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34317-0x1012b596d9a0003, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:22:28,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34387-0x1012b596d9a0002, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T13:22:28,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39675-0x1012b596d9a0001, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T13:22:28,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34387-0x1012b596d9a0002, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:22:28,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39675-0x1012b596d9a0001, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:22:28,563 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T13:22:28,565 INFO [master/cad571b91d50:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/cad571b91d50,44823,1733750546819 from backup master directory 2024-12-09T13:22:28,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/cad571b91d50,44823,1733750546819 2024-12-09T13:22:28,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34387-0x1012b596d9a0002, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T13:22:28,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T13:22:28,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34317-0x1012b596d9a0003, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T13:22:28,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39675-0x1012b596d9a0001, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T13:22:28,570 WARN [master/cad571b91d50:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T13:22:28,570 INFO [master/cad571b91d50:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=cad571b91d50,44823,1733750546819 2024-12-09T13:22:28,572 INFO [master/cad571b91d50:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-09T13:22:28,577 INFO [master/cad571b91d50:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-09T13:22:28,667 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/hbase.id] with ID: 08bed1e9-22ca-42c0-b4c7-615143020766 2024-12-09T13:22:28,667 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/.tmp/hbase.id 2024-12-09T13:22:28,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741826_1002 (size=42) 2024-12-09T13:22:28,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741826_1002 (size=42) 2024-12-09T13:22:28,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741826_1002 (size=42) 2024-12-09T13:22:29,207 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/.tmp/hbase.id]:[hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/hbase.id] 2024-12-09T13:22:29,305 INFO [master/cad571b91d50:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T13:22:29,315 INFO [master/cad571b91d50:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-09T13:22:29,358 INFO [master/cad571b91d50:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 34ms. 2024-12-09T13:22:29,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34317-0x1012b596d9a0003, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:22:29,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34387-0x1012b596d9a0002, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:22:29,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39675-0x1012b596d9a0001, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:22:29,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:22:29,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741827_1003 (size=196) 2024-12-09T13:22:29,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741827_1003 (size=196) 2024-12-09T13:22:29,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741827_1003 (size=196) 2024-12-09T13:22:29,498 INFO [master/cad571b91d50:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T13:22:29,500 INFO [master/cad571b91d50:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T13:22:29,513 INFO [master/cad571b91d50:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:22:29,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741828_1004 (size=1189) 2024-12-09T13:22:29,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741828_1004 (size=1189) 2024-12-09T13:22:29,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741828_1004 (size=1189) 2024-12-09T13:22:29,651 INFO [master/cad571b91d50:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/MasterData/data/master/store 2024-12-09T13:22:29,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741829_1005 (size=34) 2024-12-09T13:22:29,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741829_1005 (size=34) 2024-12-09T13:22:29,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741829_1005 (size=34) 2024-12-09T13:22:29,710 INFO [master/cad571b91d50:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T13:22:29,714 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:29,715 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T13:22:29,715 INFO [master/cad571b91d50:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T13:22:29,716 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T13:22:29,718 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T13:22:29,718 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T13:22:29,718 INFO [master/cad571b91d50:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T13:22:29,720 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733750549715Disabling compacts and flushes for region at 1733750549715Disabling writes for close at 1733750549718 (+3 ms)Writing region close event to WAL at 1733750549718Closed at 1733750549718 2024-12-09T13:22:29,722 WARN [master/cad571b91d50:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/MasterData/data/master/store/.initializing 2024-12-09T13:22:29,723 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/MasterData/WALs/cad571b91d50,44823,1733750546819 2024-12-09T13:22:29,736 INFO [master/cad571b91d50:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T13:22:29,765 INFO [master/cad571b91d50:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cad571b91d50%2C44823%2C1733750546819, suffix=, logDir=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/MasterData/WALs/cad571b91d50,44823,1733750546819, archiveDir=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/MasterData/oldWALs, maxLogs=10 2024-12-09T13:22:29,814 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/MasterData/WALs/cad571b91d50,44823,1733750546819/cad571b91d50%2C44823%2C1733750546819.1733750549775, exclude list is [], retry=0 2024-12-09T13:22:29,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:22:29,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:22:29,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:22:29,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:22:29,866 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-09T13:22:29,933 INFO [master/cad571b91d50:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/MasterData/WALs/cad571b91d50,44823,1733750546819/cad571b91d50%2C44823%2C1733750546819.1733750549775 2024-12-09T13:22:29,935 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34135:34135),(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34323:34323)] 2024-12-09T13:22:29,935 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T13:22:29,936 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:29,942 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T13:22:29,943 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T13:22:30,043 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T13:22:30,087 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T13:22:30,094 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:30,105 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T13:22:30,107 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T13:22:30,118 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T13:22:30,119 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:30,127 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:30,127 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T13:22:30,136 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T13:22:30,136 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:30,141 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:30,142 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T13:22:30,149 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T13:22:30,151 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:30,152 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:30,153 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T13:22:30,158 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T13:22:30,162 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T13:22:30,169 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T13:22:30,170 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T13:22:30,174 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T13:22:30,189 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T13:22:30,207 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T13:22:30,211 INFO [master/cad571b91d50:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65637610, jitterRate=-0.02192339301109314}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T13:22:30,221 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733750549961Initializing all the Stores at 1733750549970 (+9 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733750549971 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750549974 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750549975 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750549975Cleaning up temporary data from old regions at 1733750550170 (+195 ms)Region opened successfully at 1733750550221 (+51 ms) 2024-12-09T13:22:30,225 INFO [master/cad571b91d50:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T13:22:30,276 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d62403c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cad571b91d50/172.17.0.3:0 2024-12-09T13:22:30,318 INFO [master/cad571b91d50:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-09T13:22:30,333 INFO [master/cad571b91d50:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-09T13:22:30,333 INFO [master/cad571b91d50:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-09T13:22:30,337 INFO [master/cad571b91d50:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T13:22:30,338 INFO [master/cad571b91d50:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-09T13:22:30,345 INFO [master/cad571b91d50:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-09T13:22:30,345 INFO [master/cad571b91d50:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T13:22:30,385 INFO [master/cad571b91d50:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T13:22:30,398 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T13:22:30,405 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-09T13:22:30,408 INFO [master/cad571b91d50:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T13:22:30,418 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T13:22:30,427 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-09T13:22:30,430 INFO [master/cad571b91d50:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T13:22:30,435 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T13:22:30,445 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-09T13:22:30,452 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T13:22:30,455 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T13:22:30,495 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T13:22:30,500 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T13:22:30,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T13:22:30,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34317-0x1012b596d9a0003, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T13:22:30,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:22:30,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34317-0x1012b596d9a0003, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:22:30,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39675-0x1012b596d9a0001, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T13:22:30,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39675-0x1012b596d9a0001, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:22:30,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34387-0x1012b596d9a0002, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T13:22:30,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34387-0x1012b596d9a0002, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:22:30,517 INFO [master/cad571b91d50:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=cad571b91d50,44823,1733750546819, sessionid=0x1012b596d9a0000, setting cluster-up flag (Was=false) 2024-12-09T13:22:30,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34387-0x1012b596d9a0002, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:22:30,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34317-0x1012b596d9a0003, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:22:30,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39675-0x1012b596d9a0001, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:22:30,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:22:30,589 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T13:22:30,597 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=cad571b91d50,44823,1733750546819 2024-12-09T13:22:30,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39675-0x1012b596d9a0001, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:22:30,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:22:30,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34387-0x1012b596d9a0002, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:22:30,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34317-0x1012b596d9a0003, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:22:30,656 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T13:22:30,676 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=cad571b91d50,44823,1733750546819 2024-12-09T13:22:30,691 INFO [master/cad571b91d50:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-09T13:22:30,725 INFO [RS:1;cad571b91d50:34387 {}] regionserver.HRegionServer(746): ClusterId : 08bed1e9-22ca-42c0-b4c7-615143020766 2024-12-09T13:22:30,727 INFO [RS:0;cad571b91d50:39675 {}] regionserver.HRegionServer(746): ClusterId : 08bed1e9-22ca-42c0-b4c7-615143020766 2024-12-09T13:22:30,729 DEBUG [RS:1;cad571b91d50:34387 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T13:22:30,730 DEBUG [RS:0;cad571b91d50:39675 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T13:22:30,747 INFO [RS:2;cad571b91d50:34317 {}] regionserver.HRegionServer(746): ClusterId : 08bed1e9-22ca-42c0-b4c7-615143020766 2024-12-09T13:22:30,747 DEBUG [RS:2;cad571b91d50:34317 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T13:22:30,768 DEBUG [RS:0;cad571b91d50:39675 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T13:22:30,769 DEBUG [RS:0;cad571b91d50:39675 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T13:22:30,777 DEBUG [RS:1;cad571b91d50:34387 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T13:22:30,777 DEBUG [RS:1;cad571b91d50:34387 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T13:22:30,779 DEBUG [RS:2;cad571b91d50:34317 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T13:22:30,779 DEBUG [RS:2;cad571b91d50:34317 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T13:22:30,795 DEBUG [RS:0;cad571b91d50:39675 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T13:22:30,796 DEBUG [RS:0;cad571b91d50:39675 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c280a4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cad571b91d50/172.17.0.3:0 2024-12-09T13:22:30,812 DEBUG [RS:1;cad571b91d50:34387 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T13:22:30,813 DEBUG [RS:1;cad571b91d50:34387 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b20680e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cad571b91d50/172.17.0.3:0 2024-12-09T13:22:30,824 DEBUG [RS:0;cad571b91d50:39675 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;cad571b91d50:39675 2024-12-09T13:22:30,831 DEBUG [RS:2;cad571b91d50:34317 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T13:22:30,831 DEBUG [RS:2;cad571b91d50:34317 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@446c269e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cad571b91d50/172.17.0.3:0 2024-12-09T13:22:30,836 INFO [RS:0;cad571b91d50:39675 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T13:22:30,836 INFO [RS:0;cad571b91d50:39675 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T13:22:30,836 DEBUG [RS:0;cad571b91d50:39675 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T13:22:30,848 INFO [RS:0;cad571b91d50:39675 {}] regionserver.HRegionServer(2659): reportForDuty to master=cad571b91d50,44823,1733750546819 with port=39675, startcode=1733750548162 2024-12-09T13:22:30,848 DEBUG [RS:1;cad571b91d50:34387 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;cad571b91d50:34387 2024-12-09T13:22:30,849 INFO [RS:1;cad571b91d50:34387 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T13:22:30,849 INFO [RS:1;cad571b91d50:34387 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T13:22:30,850 DEBUG [RS:1;cad571b91d50:34387 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T13:22:30,846 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T13:22:30,868 INFO [master/cad571b91d50:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-09T13:22:30,870 DEBUG [RS:2;cad571b91d50:34317 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;cad571b91d50:34317 2024-12-09T13:22:30,871 INFO [RS:2;cad571b91d50:34317 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T13:22:30,871 INFO [RS:2;cad571b91d50:34317 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T13:22:30,871 DEBUG [RS:2;cad571b91d50:34317 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T13:22:30,875 INFO [RS:1;cad571b91d50:34387 {}] regionserver.HRegionServer(2659): reportForDuty to master=cad571b91d50,44823,1733750546819 with port=34387, startcode=1733750548376 2024-12-09T13:22:30,880 INFO [RS:2;cad571b91d50:34317 {}] regionserver.HRegionServer(2659): reportForDuty to master=cad571b91d50,44823,1733750546819 with port=34317, startcode=1733750548453 2024-12-09T13:22:30,885 INFO [master/cad571b91d50:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T13:22:30,892 DEBUG [RS:1;cad571b91d50:34387 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T13:22:30,892 DEBUG [RS:2;cad571b91d50:34317 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T13:22:30,894 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: cad571b91d50,44823,1733750546819 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T13:22:30,898 DEBUG [RS:0;cad571b91d50:39675 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T13:22:30,907 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/cad571b91d50:0, corePoolSize=5, maxPoolSize=5 2024-12-09T13:22:30,908 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/cad571b91d50:0, corePoolSize=5, maxPoolSize=5 2024-12-09T13:22:30,908 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/cad571b91d50:0, corePoolSize=5, maxPoolSize=5 2024-12-09T13:22:30,908 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/cad571b91d50:0, corePoolSize=5, maxPoolSize=5 2024-12-09T13:22:30,908 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/cad571b91d50:0, corePoolSize=10, maxPoolSize=10 2024-12-09T13:22:30,908 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:30,908 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/cad571b91d50:0, corePoolSize=2, maxPoolSize=2 2024-12-09T13:22:30,909 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:30,963 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T13:22:30,964 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-09T13:22:30,983 INFO [master/cad571b91d50:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733750580983 2024-12-09T13:22:30,985 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:30,985 INFO [master/cad571b91d50:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T13:22:30,987 INFO [master/cad571b91d50:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T13:22:30,992 INFO [master/cad571b91d50:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T13:22:30,992 INFO [master/cad571b91d50:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T13:22:30,992 INFO [master/cad571b91d50:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T13:22:30,993 INFO [master/cad571b91d50:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T13:22:30,985 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T13:22:31,012 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40789, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T13:22:31,021 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44823 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T13:22:31,022 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60321, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T13:22:31,023 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40251, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T13:22:31,003 INFO [master/cad571b91d50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,036 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44823 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T13:22:31,037 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44823 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T13:22:31,059 INFO [master/cad571b91d50:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T13:22:31,060 INFO [master/cad571b91d50:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T13:22:31,067 INFO [master/cad571b91d50:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T13:22:31,075 INFO [master/cad571b91d50:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T13:22:31,075 INFO [master/cad571b91d50:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T13:22:31,086 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/cad571b91d50:0:becomeActiveMaster-HFileCleaner.large.0-1733750551077,5,FailOnTimeoutGroup] 2024-12-09T13:22:31,096 DEBUG [RS:0;cad571b91d50:39675 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-09T13:22:31,096 DEBUG [RS:1;cad571b91d50:34387 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-09T13:22:31,096 DEBUG [RS:2;cad571b91d50:34317 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-09T13:22:31,096 WARN [RS:2;cad571b91d50:34317 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-09T13:22:31,103 WARN [RS:1;cad571b91d50:34387 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-09T13:22:31,102 WARN [RS:0;cad571b91d50:39675 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-09T13:22:31,098 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/cad571b91d50:0:becomeActiveMaster-HFileCleaner.small.0-1733750551087,5,FailOnTimeoutGroup] 2024-12-09T13:22:31,106 INFO [master/cad571b91d50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,107 INFO [master/cad571b91d50:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T13:22:31,144 INFO [master/cad571b91d50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,144 INFO [master/cad571b91d50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741831_1007 (size=1321) 2024-12-09T13:22:31,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741831_1007 (size=1321) 2024-12-09T13:22:31,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741831_1007 (size=1321) 2024-12-09T13:22:31,163 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-09T13:22:31,164 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1 2024-12-09T13:22:31,208 INFO [RS:2;cad571b91d50:34317 {}] regionserver.HRegionServer(2659): reportForDuty to master=cad571b91d50,44823,1733750546819 with port=34317, startcode=1733750548453 2024-12-09T13:22:31,211 INFO [RS:1;cad571b91d50:34387 {}] regionserver.HRegionServer(2659): reportForDuty to master=cad571b91d50,44823,1733750546819 with port=34387, startcode=1733750548376 2024-12-09T13:22:31,211 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44823 {}] master.ServerManager(363): Checking decommissioned status of RegionServer cad571b91d50,34317,1733750548453 2024-12-09T13:22:31,214 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44823 {}] master.ServerManager(517): Registering regionserver=cad571b91d50,34317,1733750548453 2024-12-09T13:22:31,224 INFO [RS:0;cad571b91d50:39675 {}] regionserver.HRegionServer(2659): reportForDuty to master=cad571b91d50,44823,1733750546819 with port=39675, startcode=1733750548162 2024-12-09T13:22:31,227 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44823 {}] master.ServerManager(363): Checking decommissioned status of RegionServer cad571b91d50,34387,1733750548376 2024-12-09T13:22:31,230 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44823 {}] master.ServerManager(517): Registering regionserver=cad571b91d50,34387,1733750548376 2024-12-09T13:22:31,232 DEBUG [RS:2;cad571b91d50:34317 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1 2024-12-09T13:22:31,232 DEBUG [RS:2;cad571b91d50:34317 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41625 2024-12-09T13:22:31,232 DEBUG [RS:2;cad571b91d50:34317 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T13:22:31,244 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44823 {}] master.ServerManager(363): Checking decommissioned status of RegionServer cad571b91d50,39675,1733750548162 2024-12-09T13:22:31,244 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44823 {}] master.ServerManager(517): Registering regionserver=cad571b91d50,39675,1733750548162 2024-12-09T13:22:31,244 DEBUG [RS:1;cad571b91d50:34387 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1 2024-12-09T13:22:31,244 DEBUG [RS:1;cad571b91d50:34387 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41625 2024-12-09T13:22:31,244 DEBUG [RS:1;cad571b91d50:34387 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T13:22:31,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T13:22:31,253 DEBUG [RS:0;cad571b91d50:39675 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1 2024-12-09T13:22:31,253 DEBUG [RS:0;cad571b91d50:39675 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41625 2024-12-09T13:22:31,253 DEBUG [RS:0;cad571b91d50:39675 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T13:22:31,254 DEBUG [RS:2;cad571b91d50:34317 {}] zookeeper.ZKUtil(111): regionserver:34317-0x1012b596d9a0003, quorum=127.0.0.1:53371, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/cad571b91d50,34317,1733750548453 2024-12-09T13:22:31,255 WARN [RS:2;cad571b91d50:34317 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T13:22:31,255 INFO [RS:2;cad571b91d50:34317 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:22:31,255 DEBUG [RS:2;cad571b91d50:34317 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34317,1733750548453 2024-12-09T13:22:31,263 DEBUG [RS:0;cad571b91d50:39675 {}] zookeeper.ZKUtil(111): regionserver:39675-0x1012b596d9a0001, quorum=127.0.0.1:53371, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/cad571b91d50,39675,1733750548162 2024-12-09T13:22:31,270 DEBUG [RS:1;cad571b91d50:34387 {}] zookeeper.ZKUtil(111): regionserver:34387-0x1012b596d9a0002, quorum=127.0.0.1:53371, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/cad571b91d50,34387,1733750548376 2024-12-09T13:22:31,270 WARN [RS:1;cad571b91d50:34387 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T13:22:31,271 INFO [RS:1;cad571b91d50:34387 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:22:31,271 DEBUG [RS:1;cad571b91d50:34387 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34387,1733750548376 2024-12-09T13:22:31,272 WARN [RS:0;cad571b91d50:39675 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T13:22:31,272 INFO [RS:0;cad571b91d50:39675 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:22:31,272 DEBUG [RS:0;cad571b91d50:39675 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,39675,1733750548162 2024-12-09T13:22:31,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741832_1008 (size=32) 2024-12-09T13:22:31,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741832_1008 (size=32) 2024-12-09T13:22:31,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741832_1008 (size=32) 2024-12-09T13:22:31,281 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [cad571b91d50,34317,1733750548453] 2024-12-09T13:22:31,284 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [cad571b91d50,34387,1733750548376] 2024-12-09T13:22:31,284 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [cad571b91d50,39675,1733750548162] 2024-12-09T13:22:31,282 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:31,296 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T13:22:31,301 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T13:22:31,301 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:31,307 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T13:22:31,308 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T13:22:31,323 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T13:22:31,323 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:31,326 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T13:22:31,327 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T13:22:31,347 INFO [RS:1;cad571b91d50:34387 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T13:22:31,350 INFO [RS:2;cad571b91d50:34317 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T13:22:31,351 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T13:22:31,351 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:31,359 INFO [RS:0;cad571b91d50:39675 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T13:22:31,365 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T13:22:31,366 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T13:22:31,380 INFO [RS:0;cad571b91d50:39675 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T13:22:31,381 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T13:22:31,381 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:31,383 INFO [RS:2;cad571b91d50:34317 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T13:22:31,387 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T13:22:31,388 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T13:22:31,390 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/hbase/meta/1588230740 2024-12-09T13:22:31,391 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/hbase/meta/1588230740 2024-12-09T13:22:31,395 INFO [RS:1;cad571b91d50:34387 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T13:22:31,395 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T13:22:31,395 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T13:22:31,401 INFO [RS:1;cad571b91d50:34387 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T13:22:31,401 INFO [RS:1;cad571b91d50:34387 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,406 INFO [RS:0;cad571b91d50:39675 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T13:22:31,406 INFO [RS:0;cad571b91d50:39675 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,407 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T13:22:31,407 INFO [RS:2;cad571b91d50:34317 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T13:22:31,407 INFO [RS:2;cad571b91d50:34317 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,411 INFO [RS:0;cad571b91d50:39675 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T13:22:31,414 INFO [RS:1;cad571b91d50:34387 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T13:22:31,419 INFO [RS:2;cad571b91d50:34317 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T13:22:31,422 INFO [RS:0;cad571b91d50:39675 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T13:22:31,425 INFO [RS:1;cad571b91d50:34387 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T13:22:31,426 INFO [RS:2;cad571b91d50:34317 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T13:22:31,427 INFO [RS:1;cad571b91d50:34387 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,427 INFO [RS:0;cad571b91d50:39675 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,428 DEBUG [RS:0;cad571b91d50:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,428 DEBUG [RS:1;cad571b91d50:34387 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,428 DEBUG [RS:0;cad571b91d50:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,428 DEBUG [RS:1;cad571b91d50:34387 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,428 DEBUG [RS:0;cad571b91d50:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,428 DEBUG [RS:1;cad571b91d50:34387 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,428 DEBUG [RS:0;cad571b91d50:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,428 DEBUG [RS:1;cad571b91d50:34387 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,428 DEBUG [RS:0;cad571b91d50:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,428 DEBUG [RS:1;cad571b91d50:34387 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,429 DEBUG [RS:0;cad571b91d50:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/cad571b91d50:0, corePoolSize=2, maxPoolSize=2 2024-12-09T13:22:31,429 DEBUG [RS:1;cad571b91d50:34387 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/cad571b91d50:0, corePoolSize=2, maxPoolSize=2 2024-12-09T13:22:31,429 DEBUG [RS:0;cad571b91d50:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,429 DEBUG [RS:1;cad571b91d50:34387 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,429 DEBUG [RS:0;cad571b91d50:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,429 DEBUG [RS:1;cad571b91d50:34387 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,429 DEBUG [RS:0;cad571b91d50:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,429 DEBUG [RS:1;cad571b91d50:34387 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,429 DEBUG [RS:0;cad571b91d50:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,429 DEBUG [RS:1;cad571b91d50:34387 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,429 DEBUG [RS:0;cad571b91d50:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,430 DEBUG [RS:1;cad571b91d50:34387 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,430 DEBUG [RS:0;cad571b91d50:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,430 DEBUG [RS:1;cad571b91d50:34387 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,430 DEBUG [RS:0;cad571b91d50:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/cad571b91d50:0, corePoolSize=3, maxPoolSize=3 2024-12-09T13:22:31,430 DEBUG [RS:1;cad571b91d50:34387 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/cad571b91d50:0, corePoolSize=3, maxPoolSize=3 2024-12-09T13:22:31,430 DEBUG [RS:0;cad571b91d50:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/cad571b91d50:0, corePoolSize=3, maxPoolSize=3 2024-12-09T13:22:31,430 DEBUG [RS:1;cad571b91d50:34387 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/cad571b91d50:0, corePoolSize=3, maxPoolSize=3 2024-12-09T13:22:31,430 INFO [RS:2;cad571b91d50:34317 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,430 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T13:22:31,431 DEBUG [RS:2;cad571b91d50:34317 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,431 DEBUG [RS:2;cad571b91d50:34317 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,431 DEBUG [RS:2;cad571b91d50:34317 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,431 DEBUG [RS:2;cad571b91d50:34317 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,431 DEBUG [RS:2;cad571b91d50:34317 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,431 DEBUG [RS:2;cad571b91d50:34317 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/cad571b91d50:0, corePoolSize=2, maxPoolSize=2 2024-12-09T13:22:31,432 DEBUG [RS:2;cad571b91d50:34317 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,432 DEBUG [RS:2;cad571b91d50:34317 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,432 DEBUG [RS:2;cad571b91d50:34317 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,432 DEBUG [RS:2;cad571b91d50:34317 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,432 DEBUG [RS:2;cad571b91d50:34317 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,433 DEBUG [RS:2;cad571b91d50:34317 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/cad571b91d50:0, corePoolSize=1, maxPoolSize=1 2024-12-09T13:22:31,433 DEBUG [RS:2;cad571b91d50:34317 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/cad571b91d50:0, corePoolSize=3, maxPoolSize=3 2024-12-09T13:22:31,433 DEBUG [RS:2;cad571b91d50:34317 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/cad571b91d50:0, corePoolSize=3, maxPoolSize=3 2024-12-09T13:22:31,453 INFO [RS:1;cad571b91d50:34387 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,453 INFO [RS:1;cad571b91d50:34387 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,453 INFO [RS:1;cad571b91d50:34387 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,453 INFO [RS:1;cad571b91d50:34387 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,453 INFO [RS:1;cad571b91d50:34387 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,454 INFO [RS:1;cad571b91d50:34387 {}] hbase.ChoreService(168): Chore ScheduledChore name=cad571b91d50,34387,1733750548376-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T13:22:31,466 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T13:22:31,470 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63946096, jitterRate=-0.047128915786743164}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T13:22:31,473 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733750551287Initializing all the Stores at 1733750551294 (+7 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733750551294Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733750551296 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750551298 (+2 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733750551298Cleaning up temporary data from old regions at 1733750551395 (+97 ms)Region opened successfully at 1733750551472 (+77 ms) 2024-12-09T13:22:31,473 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T13:22:31,473 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T13:22:31,473 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T13:22:31,473 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T13:22:31,473 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T13:22:31,477 INFO [RS:2;cad571b91d50:34317 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,477 INFO [RS:2;cad571b91d50:34317 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,477 INFO [RS:2;cad571b91d50:34317 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,478 INFO [RS:2;cad571b91d50:34317 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,478 INFO [RS:2;cad571b91d50:34317 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,478 INFO [RS:2;cad571b91d50:34317 {}] hbase.ChoreService(168): Chore ScheduledChore name=cad571b91d50,34317,1733750548453-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T13:22:31,486 INFO [RS:0;cad571b91d50:39675 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,487 INFO [RS:0;cad571b91d50:39675 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,487 INFO [RS:0;cad571b91d50:39675 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,487 INFO [RS:0;cad571b91d50:39675 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,490 INFO [RS:0;cad571b91d50:39675 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,490 INFO [RS:0;cad571b91d50:39675 {}] hbase.ChoreService(168): Chore ScheduledChore name=cad571b91d50,39675,1733750548162-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T13:22:31,493 INFO [RS:1;cad571b91d50:34387 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T13:22:31,497 INFO [RS:1;cad571b91d50:34387 {}] hbase.ChoreService(168): Chore ScheduledChore name=cad571b91d50,34387,1733750548376-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,497 INFO [RS:1;cad571b91d50:34387 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,497 INFO [RS:1;cad571b91d50:34387 {}] regionserver.Replication(171): cad571b91d50,34387,1733750548376 started 2024-12-09T13:22:31,515 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T13:22:31,515 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733750551473Disabling compacts and flushes for region at 1733750551473Disabling writes for close at 1733750551473Writing region close event to WAL at 1733750551514 (+41 ms)Closed at 1733750551515 (+1 ms) 2024-12-09T13:22:31,520 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T13:22:31,520 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-09T13:22:31,529 INFO [RS:2;cad571b91d50:34317 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T13:22:31,529 INFO [RS:2;cad571b91d50:34317 {}] hbase.ChoreService(168): Chore ScheduledChore name=cad571b91d50,34317,1733750548453-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,530 INFO [RS:2;cad571b91d50:34317 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,531 INFO [RS:2;cad571b91d50:34317 {}] regionserver.Replication(171): cad571b91d50,34317,1733750548453 started 2024-12-09T13:22:31,545 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T13:22:31,638 INFO [RS:0;cad571b91d50:39675 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T13:22:31,641 INFO [RS:0;cad571b91d50:39675 {}] hbase.ChoreService(168): Chore ScheduledChore name=cad571b91d50,39675,1733750548162-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,641 INFO [RS:0;cad571b91d50:39675 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,641 INFO [RS:0;cad571b91d50:39675 {}] regionserver.Replication(171): cad571b91d50,39675,1733750548162 started 2024-12-09T13:22:31,645 INFO [RS:1;cad571b91d50:34387 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,652 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T13:22:31,653 INFO [RS:1;cad571b91d50:34387 {}] regionserver.HRegionServer(1482): Serving as cad571b91d50,34387,1733750548376, RpcServer on cad571b91d50/172.17.0.3:34387, sessionid=0x1012b596d9a0002 2024-12-09T13:22:31,654 DEBUG [RS:1;cad571b91d50:34387 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T13:22:31,655 DEBUG [RS:1;cad571b91d50:34387 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager cad571b91d50,34387,1733750548376 2024-12-09T13:22:31,655 DEBUG [RS:1;cad571b91d50:34387 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cad571b91d50,34387,1733750548376' 2024-12-09T13:22:31,655 DEBUG [RS:1;cad571b91d50:34387 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T13:22:31,656 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T13:22:31,662 DEBUG [RS:1;cad571b91d50:34387 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T13:22:31,664 DEBUG [RS:1;cad571b91d50:34387 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T13:22:31,664 DEBUG [RS:1;cad571b91d50:34387 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T13:22:31,664 DEBUG [RS:1;cad571b91d50:34387 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager cad571b91d50,34387,1733750548376 2024-12-09T13:22:31,664 DEBUG [RS:1;cad571b91d50:34387 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cad571b91d50,34387,1733750548376' 2024-12-09T13:22:31,664 DEBUG [RS:1;cad571b91d50:34387 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T13:22:31,667 DEBUG [RS:1;cad571b91d50:34387 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T13:22:31,678 DEBUG [RS:1;cad571b91d50:34387 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T13:22:31,679 INFO [RS:1;cad571b91d50:34387 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T13:22:31,679 INFO [RS:1;cad571b91d50:34387 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T13:22:31,683 INFO [RS:2;cad571b91d50:34317 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,683 INFO [RS:2;cad571b91d50:34317 {}] regionserver.HRegionServer(1482): Serving as cad571b91d50,34317,1733750548453, RpcServer on cad571b91d50/172.17.0.3:34317, sessionid=0x1012b596d9a0003 2024-12-09T13:22:31,684 DEBUG [RS:2;cad571b91d50:34317 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T13:22:31,685 DEBUG [RS:2;cad571b91d50:34317 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager cad571b91d50,34317,1733750548453 2024-12-09T13:22:31,685 DEBUG [RS:2;cad571b91d50:34317 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cad571b91d50,34317,1733750548453' 2024-12-09T13:22:31,685 DEBUG [RS:2;cad571b91d50:34317 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T13:22:31,694 DEBUG [RS:2;cad571b91d50:34317 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T13:22:31,697 DEBUG [RS:2;cad571b91d50:34317 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T13:22:31,697 DEBUG [RS:2;cad571b91d50:34317 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T13:22:31,697 DEBUG [RS:2;cad571b91d50:34317 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager cad571b91d50,34317,1733750548453 2024-12-09T13:22:31,697 DEBUG [RS:2;cad571b91d50:34317 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cad571b91d50,34317,1733750548453' 2024-12-09T13:22:31,697 DEBUG [RS:2;cad571b91d50:34317 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T13:22:31,703 DEBUG [RS:2;cad571b91d50:34317 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T13:22:31,704 INFO [RS:0;cad571b91d50:39675 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:31,705 INFO [RS:0;cad571b91d50:39675 {}] regionserver.HRegionServer(1482): Serving as cad571b91d50,39675,1733750548162, RpcServer on cad571b91d50/172.17.0.3:39675, sessionid=0x1012b596d9a0001 2024-12-09T13:22:31,705 DEBUG [RS:0;cad571b91d50:39675 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T13:22:31,705 DEBUG [RS:0;cad571b91d50:39675 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager cad571b91d50,39675,1733750548162 2024-12-09T13:22:31,705 DEBUG [RS:0;cad571b91d50:39675 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cad571b91d50,39675,1733750548162' 2024-12-09T13:22:31,705 DEBUG [RS:0;cad571b91d50:39675 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T13:22:31,714 DEBUG [RS:0;cad571b91d50:39675 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T13:22:31,722 DEBUG [RS:2;cad571b91d50:34317 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T13:22:31,723 INFO [RS:2;cad571b91d50:34317 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T13:22:31,723 INFO [RS:2;cad571b91d50:34317 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T13:22:31,724 DEBUG [RS:0;cad571b91d50:39675 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T13:22:31,724 DEBUG [RS:0;cad571b91d50:39675 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T13:22:31,724 DEBUG [RS:0;cad571b91d50:39675 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager cad571b91d50,39675,1733750548162 2024-12-09T13:22:31,724 DEBUG [RS:0;cad571b91d50:39675 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cad571b91d50,39675,1733750548162' 2024-12-09T13:22:31,724 DEBUG [RS:0;cad571b91d50:39675 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T13:22:31,733 DEBUG [RS:0;cad571b91d50:39675 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T13:22:31,751 DEBUG [RS:0;cad571b91d50:39675 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T13:22:31,751 INFO [RS:0;cad571b91d50:39675 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T13:22:31,751 INFO [RS:0;cad571b91d50:39675 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T13:22:31,787 INFO [RS:1;cad571b91d50:34387 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T13:22:31,795 INFO [RS:1;cad571b91d50:34387 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cad571b91d50%2C34387%2C1733750548376, suffix=, logDir=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34387,1733750548376, archiveDir=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/oldWALs, maxLogs=32 2024-12-09T13:22:31,811 WARN [cad571b91d50:44823 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T13:22:31,817 DEBUG [RS:1;cad571b91d50:34387 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34387,1733750548376/cad571b91d50%2C34387%2C1733750548376.1733750551798, exclude list is [], retry=0 2024-12-09T13:22:31,823 INFO [RS:2;cad571b91d50:34317 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T13:22:31,830 INFO [RS:2;cad571b91d50:34317 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cad571b91d50%2C34317%2C1733750548453, suffix=, logDir=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34317,1733750548453, archiveDir=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/oldWALs, maxLogs=32 2024-12-09T13:22:31,832 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:22:31,833 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:22:31,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:22:31,853 INFO [RS:0;cad571b91d50:39675 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T13:22:31,856 DEBUG [RS:2;cad571b91d50:34317 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34317,1733750548453/cad571b91d50%2C34317%2C1733750548453.1733750551832, exclude list is [], retry=0 2024-12-09T13:22:31,862 INFO [RS:0;cad571b91d50:39675 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cad571b91d50%2C39675%2C1733750548162, suffix=, logDir=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,39675,1733750548162, archiveDir=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/oldWALs, maxLogs=32 2024-12-09T13:22:31,866 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:22:31,866 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:22:31,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:22:31,880 INFO [RS:1;cad571b91d50:34387 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34387,1733750548376/cad571b91d50%2C34387%2C1733750548376.1733750551798 2024-12-09T13:22:31,884 DEBUG [RS:0;cad571b91d50:39675 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,39675,1733750548162/cad571b91d50%2C39675%2C1733750548162.1733750551864, exclude list is [], retry=0 2024-12-09T13:22:31,891 INFO [RS:2;cad571b91d50:34317 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34317,1733750548453/cad571b91d50%2C34317%2C1733750548453.1733750551832 2024-12-09T13:22:31,891 DEBUG [RS:1;cad571b91d50:34387 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34323:34323),(127.0.0.1/127.0.0.1:34135:34135)] 2024-12-09T13:22:31,896 DEBUG [RS:2;cad571b91d50:34317 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34323:34323),(127.0.0.1/127.0.0.1:34135:34135)] 2024-12-09T13:22:31,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:22:31,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:22:31,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:22:31,956 INFO [RS:0;cad571b91d50:39675 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,39675,1733750548162/cad571b91d50%2C39675%2C1733750548162.1733750551864 2024-12-09T13:22:31,958 DEBUG [RS:0;cad571b91d50:39675 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34323:34323),(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34135:34135)] 2024-12-09T13:22:32,064 DEBUG [cad571b91d50:44823 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-09T13:22:32,112 DEBUG [cad571b91d50:44823 {}] balancer.BalancerClusterState(204): Hosts are {cad571b91d50=0} racks are {/default-rack=0} 2024-12-09T13:22:32,131 DEBUG [cad571b91d50:44823 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T13:22:32,132 DEBUG [cad571b91d50:44823 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T13:22:32,132 DEBUG [cad571b91d50:44823 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-09T13:22:32,132 DEBUG [cad571b91d50:44823 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T13:22:32,132 DEBUG [cad571b91d50:44823 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T13:22:32,132 DEBUG [cad571b91d50:44823 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-09T13:22:32,132 INFO [cad571b91d50:44823 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T13:22:32,132 INFO [cad571b91d50:44823 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T13:22:32,132 INFO [cad571b91d50:44823 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-09T13:22:32,132 DEBUG [cad571b91d50:44823 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T13:22:32,159 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=cad571b91d50,34317,1733750548453 2024-12-09T13:22:32,175 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as cad571b91d50,34317,1733750548453, state=OPENING 2024-12-09T13:22:32,182 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T13:22:32,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34317-0x1012b596d9a0003, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:22:32,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34387-0x1012b596d9a0002, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:22:32,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:22:32,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39675-0x1012b596d9a0001, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:22:32,192 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T13:22:32,192 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T13:22:32,194 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T13:22:32,195 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T13:22:32,197 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T13:22:32,204 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=cad571b91d50,34317,1733750548453}] 2024-12-09T13:22:32,384 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T13:22:32,386 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42689, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T13:22:32,402 INFO [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T13:22:32,403 INFO [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:22:32,404 INFO [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T13:22:32,408 INFO [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cad571b91d50%2C34317%2C1733750548453.meta, suffix=.meta, logDir=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34317,1733750548453, archiveDir=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/oldWALs, maxLogs=32 2024-12-09T13:22:32,425 DEBUG [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34317,1733750548453/cad571b91d50%2C34317%2C1733750548453.meta.1733750552410.meta, exclude list is [], retry=0 2024-12-09T13:22:32,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:22:32,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:22:32,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:22:32,440 INFO [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34317,1733750548453/cad571b91d50%2C34317%2C1733750548453.meta.1733750552410.meta 2024-12-09T13:22:32,441 DEBUG [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34323:34323),(127.0.0.1/127.0.0.1:34135:34135),(127.0.0.1/127.0.0.1:44161:44161)] 2024-12-09T13:22:32,441 DEBUG [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T13:22:32,443 DEBUG [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T13:22:32,447 DEBUG [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T13:22:32,453 INFO [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T13:22:32,460 DEBUG [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T13:22:32,461 DEBUG [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:32,461 DEBUG [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T13:22:32,461 DEBUG [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T13:22:32,467 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T13:22:32,468 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T13:22:32,468 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:32,469 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T13:22:32,470 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T13:22:32,471 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T13:22:32,471 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:32,472 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T13:22:32,473 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T13:22:32,474 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T13:22:32,474 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:32,475 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T13:22:32,475 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T13:22:32,477 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T13:22:32,477 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:32,478 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T13:22:32,478 DEBUG [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T13:22:32,480 DEBUG [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/hbase/meta/1588230740 2024-12-09T13:22:32,483 DEBUG [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/hbase/meta/1588230740 2024-12-09T13:22:32,495 DEBUG [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T13:22:32,495 DEBUG [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T13:22:32,496 DEBUG [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T13:22:32,512 DEBUG [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T13:22:32,520 INFO [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68600552, jitterRate=0.022227883338928223}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T13:22:32,520 DEBUG [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T13:22:32,522 DEBUG [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733750552462Writing region info on filesystem at 1733750552462Initializing all the Stores at 1733750552464 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733750552464Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733750552466 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750552466Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733750552466Cleaning up temporary data from old regions at 1733750552495 (+29 ms)Running coprocessor post-open hooks at 1733750552520 (+25 ms)Region opened successfully at 1733750552521 (+1 ms) 2024-12-09T13:22:32,531 INFO [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733750552373 2024-12-09T13:22:32,545 DEBUG [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T13:22:32,545 INFO [RS_OPEN_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T13:22:32,548 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=cad571b91d50,34317,1733750548453 2024-12-09T13:22:32,555 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as cad571b91d50,34317,1733750548453, state=OPEN 2024-12-09T13:22:32,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34317-0x1012b596d9a0003, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T13:22:32,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T13:22:32,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34387-0x1012b596d9a0002, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T13:22:32,559 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T13:22:32,559 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T13:22:32,559 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T13:22:32,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39675-0x1012b596d9a0001, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T13:22:32,559 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T13:22:32,560 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=cad571b91d50,34317,1733750548453 2024-12-09T13:22:32,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T13:22:32,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=cad571b91d50,34317,1733750548453 in 356 msec 2024-12-09T13:22:32,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T13:22:32,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.0250 sec 2024-12-09T13:22:32,585 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T13:22:32,585 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-09T13:22:32,608 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T13:22:32,610 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cad571b91d50,34317,1733750548453, seqNum=-1] 2024-12-09T13:22:32,632 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T13:22:32,635 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47123, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T13:22:32,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.9070 sec 2024-12-09T13:22:32,672 INFO [master/cad571b91d50:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733750552672, completionTime=-1 2024-12-09T13:22:32,675 INFO [master/cad571b91d50:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-09T13:22:32,676 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T13:22:32,708 INFO [master/cad571b91d50:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-09T13:22:32,709 INFO [master/cad571b91d50:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733750612708 2024-12-09T13:22:32,709 INFO [master/cad571b91d50:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733750672709 2024-12-09T13:22:32,709 INFO [master/cad571b91d50:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 33 msec 2024-12-09T13:22:32,710 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-09T13:22:32,718 INFO [master/cad571b91d50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cad571b91d50,44823,1733750546819-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:32,719 INFO [master/cad571b91d50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cad571b91d50,44823,1733750546819-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:32,719 INFO [master/cad571b91d50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cad571b91d50,44823,1733750546819-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:32,721 INFO [master/cad571b91d50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-cad571b91d50:44823, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:32,722 INFO [master/cad571b91d50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:32,723 INFO [master/cad571b91d50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:32,731 DEBUG [master/cad571b91d50:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-09T13:22:32,760 INFO [master/cad571b91d50:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.190sec 2024-12-09T13:22:32,762 INFO [master/cad571b91d50:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T13:22:32,764 INFO [master/cad571b91d50:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T13:22:32,765 INFO [master/cad571b91d50:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T13:22:32,765 INFO [master/cad571b91d50:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T13:22:32,766 INFO [master/cad571b91d50:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T13:22:32,766 INFO [master/cad571b91d50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cad571b91d50,44823,1733750546819-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T13:22:32,767 INFO [master/cad571b91d50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cad571b91d50,44823,1733750546819-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T13:22:32,772 DEBUG [master/cad571b91d50:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-09T13:22:32,773 INFO [master/cad571b91d50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T13:22:32,773 INFO [master/cad571b91d50:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cad571b91d50,44823,1733750546819-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:32,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e6a8982, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T13:22:32,850 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cad571b91d50,44823,-1 for getting cluster id 2024-12-09T13:22:32,853 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T13:22:32,865 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '08bed1e9-22ca-42c0-b4c7-615143020766' 2024-12-09T13:22:32,869 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T13:22:32,869 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "08bed1e9-22ca-42c0-b4c7-615143020766" 2024-12-09T13:22:32,869 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20878120, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T13:22:32,869 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cad571b91d50,44823,-1] 2024-12-09T13:22:32,873 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T13:22:32,875 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T13:22:32,879 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T13:22:32,882 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c41726a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T13:22:32,883 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T13:22:32,892 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cad571b91d50,34317,1733750548453, seqNum=-1] 2024-12-09T13:22:32,893 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T13:22:32,897 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T13:22:32,928 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=cad571b91d50,44823,1733750546819 2024-12-09T13:22:32,929 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:41625/hbase 2024-12-09T13:22:32,951 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=358, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=707, ProcessCount=11, AvailableMemoryMB=4052 2024-12-09T13:22:32,990 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:22:32,996 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:22:32,998 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T13:22:33,019 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-28835465, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/hregion-28835465, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:22:33,044 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-28835465/hregion-28835465.1733750553020, exclude list is [], retry=0 2024-12-09T13:22:33,055 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:22:33,057 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:22:33,061 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:22:33,095 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-28835465/hregion-28835465.1733750553020 2024-12-09T13:22:33,099 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34135:34135),(127.0.0.1/127.0.0.1:34323:34323),(127.0.0.1/127.0.0.1:44161:44161)] 2024-12-09T13:22:33,099 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 43b5a489a6ceb874873e239fae131d26, NAME => 'testReplayEditsWrittenIntoWAL,,1733750552991.43b5a489a6ceb874873e239fae131d26.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41625/hbase 2024-12-09T13:22:33,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741838_1014 (size=64) 2024-12-09T13:22:33,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741838_1014 (size=64) 2024-12-09T13:22:33,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741838_1014 (size=64) 2024-12-09T13:22:33,172 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733750552991.43b5a489a6ceb874873e239fae131d26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:33,184 INFO [StoreOpener-43b5a489a6ceb874873e239fae131d26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 43b5a489a6ceb874873e239fae131d26 2024-12-09T13:22:33,187 INFO [StoreOpener-43b5a489a6ceb874873e239fae131d26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43b5a489a6ceb874873e239fae131d26 columnFamilyName a 2024-12-09T13:22:33,187 DEBUG [StoreOpener-43b5a489a6ceb874873e239fae131d26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:33,188 INFO [StoreOpener-43b5a489a6ceb874873e239fae131d26-1 {}] regionserver.HStore(327): Store=43b5a489a6ceb874873e239fae131d26/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:33,188 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 43b5a489a6ceb874873e239fae131d26 2024-12-09T13:22:33,191 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26 2024-12-09T13:22:33,191 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26 2024-12-09T13:22:33,192 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 43b5a489a6ceb874873e239fae131d26 2024-12-09T13:22:33,192 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 43b5a489a6ceb874873e239fae131d26 2024-12-09T13:22:33,199 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 43b5a489a6ceb874873e239fae131d26 2024-12-09T13:22:33,215 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T13:22:33,216 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 43b5a489a6ceb874873e239fae131d26; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70251255, jitterRate=0.04682527482509613}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T13:22:33,217 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 43b5a489a6ceb874873e239fae131d26: Writing region info on filesystem at 1733750553173Initializing all the Stores at 1733750553184 (+11 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750553184Cleaning up temporary data from old regions at 1733750553192 (+8 ms)Region opened successfully at 1733750553217 (+25 ms) 2024-12-09T13:22:33,218 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 43b5a489a6ceb874873e239fae131d26, disabling compactions & flushes 2024-12-09T13:22:33,218 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733750552991.43b5a489a6ceb874873e239fae131d26. 2024-12-09T13:22:33,218 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733750552991.43b5a489a6ceb874873e239fae131d26. 2024-12-09T13:22:33,218 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733750552991.43b5a489a6ceb874873e239fae131d26. after waiting 0 ms 2024-12-09T13:22:33,218 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733750552991.43b5a489a6ceb874873e239fae131d26. 2024-12-09T13:22:33,218 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733750552991.43b5a489a6ceb874873e239fae131d26. 2024-12-09T13:22:33,218 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 43b5a489a6ceb874873e239fae131d26: Waiting for close lock at 1733750553218Disabling compacts and flushes for region at 1733750553218Disabling writes for close at 1733750553218Writing region close event to WAL at 1733750553218Closed at 1733750553218 2024-12-09T13:22:33,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741837_1013 (size=93) 2024-12-09T13:22:33,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741837_1013 (size=93) 2024-12-09T13:22:33,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741837_1013 (size=93) 2024-12-09T13:22:33,264 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T13:22:33,264 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-28835465:(num 1733750553020) 2024-12-09T13:22:33,266 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T13:22:33,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741839_1015 (size=369) 2024-12-09T13:22:33,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741839_1015 (size=369) 2024-12-09T13:22:33,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741839_1015 (size=369) 2024-12-09T13:22:33,335 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T13:22:33,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741840_1016 (size=231) 2024-12-09T13:22:33,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741840_1016 (size=231) 2024-12-09T13:22:33,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741840_1016 (size=231) 2024-12-09T13:22:33,403 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982/wal-1, size=369 (369bytes) 2024-12-09T13:22:33,404 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-09T13:22:33,405 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-09T13:22:33,405 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982/wal-1 2024-12-09T13:22:33,413 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982/wal-1 after 4ms 2024-12-09T13:22:33,422 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982/wal-1 took 21ms 2024-12-09T13:22:33,428 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982/wal-1 so closing down 2024-12-09T13:22:33,433 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-09T13:22:33,434 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T13:22:33,442 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26/recovered.edits/0000000000000000001-wal-1.temp 2024-12-09T13:22:33,443 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T13:22:33,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741841_1017 (size=369) 2024-12-09T13:22:33,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741841_1017 (size=369) 2024-12-09T13:22:33,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741841_1017 (size=369) 2024-12-09T13:22:33,485 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-09T13:22:33,495 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26/recovered.edits/0000000000000000002 2024-12-09T13:22:33,504 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 73 ms; skipped=0; WAL=hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-12-09T13:22:33,504 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982/wal-1, journal: Splitting hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982/wal-1, size=369 (369bytes) at 1733750553404Finishing writing output for hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982/wal-1 so closing down at 1733750553428 (+24 ms)Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26/recovered.edits/0000000000000000001-wal-1.temp at 1733750553442 (+14 ms)3 split writer threads finished at 1733750553443 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733750553485 (+42 ms)Rename recovered edits hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26/recovered.edits/0000000000000000002 at 1733750553495 (+10 ms)Processed 2 edits across 1 Regions in 73 ms; skipped=0; WAL=hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1733750553504 (+9 ms) 2024-12-09T13:22:33,537 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982/wal-2, size=231 (231bytes) 2024-12-09T13:22:33,538 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982/wal-2 2024-12-09T13:22:33,548 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982/wal-2 after 7ms 2024-12-09T13:22:33,587 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982/wal-2 took 50ms 2024-12-09T13:22:33,605 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982/wal-2 so closing down 2024-12-09T13:22:33,606 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T13:22:33,637 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-09T13:22:33,651 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26/recovered.edits/0000000000000000002-wal-2.temp 2024-12-09T13:22:33,652 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T13:22:33,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741842_1018 (size=231) 2024-12-09T13:22:33,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741842_1018 (size=231) 2024-12-09T13:22:33,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741842_1018 (size=231) 2024-12-09T13:22:33,736 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-09T13:22:33,799 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26/recovered.edits/0000000000000000002-wal-2.temp, length=231 2024-12-09T13:22:33,810 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 217 ms; skipped=0; WAL=hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-12-09T13:22:33,811 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982/wal-2, journal: Splitting hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982/wal-2, size=231 (231bytes) at 1733750553538Finishing writing output for hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982/wal-2 so closing down at 1733750553606 (+68 ms)Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26/recovered.edits/0000000000000000002-wal-2.temp at 1733750553652 (+46 ms)3 split writer threads finished at 1733750553652Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733750553737 (+85 ms)Processed 1 edits across 1 Regions in 217 ms; skipped=0; WAL=hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1733750553810 (+73 ms) 2024-12-09T13:22:33,811 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T13:22:33,831 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:22:33,855 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982/wal.1733750553832, exclude list is [], retry=0 2024-12-09T13:22:33,860 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:22:33,861 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:22:33,861 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:22:33,876 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982/wal.1733750553832 2024-12-09T13:22:33,878 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34135:34135),(127.0.0.1/127.0.0.1:34323:34323)] 2024-12-09T13:22:33,879 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 43b5a489a6ceb874873e239fae131d26, NAME => 'testReplayEditsWrittenIntoWAL,,1733750552991.43b5a489a6ceb874873e239fae131d26.', STARTKEY => '', ENDKEY => ''} 2024-12-09T13:22:33,879 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733750552991.43b5a489a6ceb874873e239fae131d26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:33,879 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 43b5a489a6ceb874873e239fae131d26 2024-12-09T13:22:33,879 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 43b5a489a6ceb874873e239fae131d26 2024-12-09T13:22:33,882 INFO [StoreOpener-43b5a489a6ceb874873e239fae131d26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 43b5a489a6ceb874873e239fae131d26 2024-12-09T13:22:33,884 INFO [StoreOpener-43b5a489a6ceb874873e239fae131d26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43b5a489a6ceb874873e239fae131d26 columnFamilyName a 2024-12-09T13:22:33,885 DEBUG [StoreOpener-43b5a489a6ceb874873e239fae131d26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:33,886 INFO [StoreOpener-43b5a489a6ceb874873e239fae131d26-1 {}] regionserver.HStore(327): Store=43b5a489a6ceb874873e239fae131d26/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:33,886 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 43b5a489a6ceb874873e239fae131d26 2024-12-09T13:22:33,887 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26 2024-12-09T13:22:33,894 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26 2024-12-09T13:22:33,895 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26/recovered.edits/0000000000000000002 2024-12-09T13:22:33,914 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26/recovered.edits/0000000000000000002 2024-12-09T13:22:33,919 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 43b5a489a6ceb874873e239fae131d26 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-09T13:22:34,005 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26/.tmp/a/9f38890ef12b41fb885784fd4716fd96 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733750553264/Put/seqid=0 2024-12-09T13:22:34,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741844_1020 (size=5170) 2024-12-09T13:22:34,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741844_1020 (size=5170) 2024-12-09T13:22:34,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741844_1020 (size=5170) 2024-12-09T13:22:34,048 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26/.tmp/a/9f38890ef12b41fb885784fd4716fd96 2024-12-09T13:22:34,117 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26/.tmp/a/9f38890ef12b41fb885784fd4716fd96 as hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26/a/9f38890ef12b41fb885784fd4716fd96 2024-12-09T13:22:34,133 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26/a/9f38890ef12b41fb885784fd4716fd96, entries=2, sequenceid=2, filesize=5.0 K 2024-12-09T13:22:34,139 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 43b5a489a6ceb874873e239fae131d26 in 220ms, sequenceid=2, compaction requested=false; wal=null 2024-12-09T13:22:34,143 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26/recovered.edits/0000000000000000002 2024-12-09T13:22:34,143 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 43b5a489a6ceb874873e239fae131d26 2024-12-09T13:22:34,144 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 43b5a489a6ceb874873e239fae131d26 2024-12-09T13:22:34,155 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 43b5a489a6ceb874873e239fae131d26 2024-12-09T13:22:34,158 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/43b5a489a6ceb874873e239fae131d26/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-09T13:22:34,160 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 43b5a489a6ceb874873e239fae131d26; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63198645, jitterRate=-0.05826680362224579}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T13:22:34,161 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 43b5a489a6ceb874873e239fae131d26: Writing region info on filesystem at 1733750553879Initializing all the Stores at 1733750553881 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750553882 (+1 ms)Obtaining lock to block concurrent updates at 1733750553919 (+37 ms)Preparing flush snapshotting stores in 43b5a489a6ceb874873e239fae131d26 at 1733750553919Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733750552991.43b5a489a6ceb874873e239fae131d26., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733750553922 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733750552991.43b5a489a6ceb874873e239fae131d26. at 1733750553922Flushing 43b5a489a6ceb874873e239fae131d26/a: creating writer at 1733750553924 (+2 ms)Flushing 43b5a489a6ceb874873e239fae131d26/a: appending metadata at 1733750553986 (+62 ms)Flushing 43b5a489a6ceb874873e239fae131d26/a: closing flushed file at 1733750553990 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a2091e9: reopening flushed file at 1733750554115 (+125 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 43b5a489a6ceb874873e239fae131d26 in 220ms, sequenceid=2, compaction requested=false; wal=null at 1733750554139 (+24 ms)Cleaning up temporary data from old regions at 1733750554144 (+5 ms)Region opened successfully at 1733750554161 (+17 ms) 2024-12-09T13:22:34,195 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=370 (was 358) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:54288 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:60570 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:41625/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:54324 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1859740653-172.17.0.3-1733750541580:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1859740653-172.17.0.3-1733750541580:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1859740653-172.17.0.3-1733750541580:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:54418 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:60690 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:54400 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=707 (was 707), ProcessCount=11 (was 11), AvailableMemoryMB=4027 (was 4052) 2024-12-09T13:22:34,207 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=370, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=707, ProcessCount=11, AvailableMemoryMB=4026 2024-12-09T13:22:34,224 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:22:34,227 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:22:34,230 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T13:22:34,234 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-80420473, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/hregion-80420473, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:22:34,251 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-80420473/hregion-80420473.1733750554235, exclude list is [], retry=0 2024-12-09T13:22:34,256 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:22:34,257 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:22:34,258 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:22:34,263 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-80420473/hregion-80420473.1733750554235 2024-12-09T13:22:34,264 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34135:34135),(127.0.0.1/127.0.0.1:34323:34323),(127.0.0.1/127.0.0.1:44161:44161)] 2024-12-09T13:22:34,264 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 44a05a8bda3ea46f351fd358f6b43061, NAME => 'testReplayEditsWrittenIntoWAL,,1733750554225.44a05a8bda3ea46f351fd358f6b43061.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41625/hbase 2024-12-09T13:22:34,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741846_1022 (size=64) 2024-12-09T13:22:34,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741846_1022 (size=64) 2024-12-09T13:22:34,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741846_1022 (size=64) 2024-12-09T13:22:34,298 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733750554225.44a05a8bda3ea46f351fd358f6b43061.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:34,301 INFO [StoreOpener-44a05a8bda3ea46f351fd358f6b43061-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 44a05a8bda3ea46f351fd358f6b43061 2024-12-09T13:22:34,309 INFO [StoreOpener-44a05a8bda3ea46f351fd358f6b43061-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44a05a8bda3ea46f351fd358f6b43061 columnFamilyName a 2024-12-09T13:22:34,309 DEBUG [StoreOpener-44a05a8bda3ea46f351fd358f6b43061-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:34,310 INFO [StoreOpener-44a05a8bda3ea46f351fd358f6b43061-1 {}] regionserver.HStore(327): Store=44a05a8bda3ea46f351fd358f6b43061/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:34,310 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 44a05a8bda3ea46f351fd358f6b43061 2024-12-09T13:22:34,311 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061 2024-12-09T13:22:34,312 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061 2024-12-09T13:22:34,313 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 44a05a8bda3ea46f351fd358f6b43061 2024-12-09T13:22:34,313 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 44a05a8bda3ea46f351fd358f6b43061 2024-12-09T13:22:34,317 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 44a05a8bda3ea46f351fd358f6b43061 2024-12-09T13:22:34,324 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T13:22:34,325 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 44a05a8bda3ea46f351fd358f6b43061; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70164107, jitterRate=0.045526668429374695}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T13:22:34,325 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 44a05a8bda3ea46f351fd358f6b43061: Writing region info on filesystem at 1733750554298Initializing all the Stores at 1733750554300 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750554300Cleaning up temporary data from old regions at 1733750554313 (+13 ms)Region opened successfully at 1733750554325 (+12 ms) 2024-12-09T13:22:34,325 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 44a05a8bda3ea46f351fd358f6b43061, disabling compactions & flushes 2024-12-09T13:22:34,325 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733750554225.44a05a8bda3ea46f351fd358f6b43061. 2024-12-09T13:22:34,325 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733750554225.44a05a8bda3ea46f351fd358f6b43061. 2024-12-09T13:22:34,325 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733750554225.44a05a8bda3ea46f351fd358f6b43061. after waiting 0 ms 2024-12-09T13:22:34,325 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733750554225.44a05a8bda3ea46f351fd358f6b43061. 2024-12-09T13:22:34,326 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733750554225.44a05a8bda3ea46f351fd358f6b43061. 2024-12-09T13:22:34,326 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 44a05a8bda3ea46f351fd358f6b43061: Waiting for close lock at 1733750554325Disabling compacts and flushes for region at 1733750554325Disabling writes for close at 1733750554325Writing region close event to WAL at 1733750554326 (+1 ms)Closed at 1733750554326 2024-12-09T13:22:34,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741845_1021 (size=93) 2024-12-09T13:22:34,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741845_1021 (size=93) 2024-12-09T13:22:34,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741845_1021 (size=93) 2024-12-09T13:22:34,358 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T13:22:34,358 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-80420473:(num 1733750554235) 2024-12-09T13:22:34,360 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T13:22:34,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741847_1023 (size=369) 2024-12-09T13:22:34,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741847_1023 (size=369) 2024-12-09T13:22:34,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741847_1023 (size=369) 2024-12-09T13:22:34,388 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T13:22:34,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741848_1024 (size=231) 2024-12-09T13:22:34,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741848_1024 (size=231) 2024-12-09T13:22:34,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741848_1024 (size=231) 2024-12-09T13:22:34,432 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223/wal-2, size=231 (231bytes) 2024-12-09T13:22:34,432 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223/wal-2 2024-12-09T13:22:34,434 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223/wal-2 after 2ms 2024-12-09T13:22:34,441 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223/wal-2 took 9ms 2024-12-09T13:22:34,453 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223/wal-2 so closing down 2024-12-09T13:22:34,454 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T13:22:34,464 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-09T13:22:34,466 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/recovered.edits/0000000000000000002-wal-2.temp 2024-12-09T13:22:34,466 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T13:22:34,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741849_1025 (size=231) 2024-12-09T13:22:34,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741849_1025 (size=231) 2024-12-09T13:22:34,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741849_1025 (size=231) 2024-12-09T13:22:34,492 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-09T13:22:34,497 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/recovered.edits/0000000000000000002 2024-12-09T13:22:34,497 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 51 ms; skipped=0; WAL=hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-12-09T13:22:34,497 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223/wal-2, journal: Splitting hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223/wal-2, size=231 (231bytes) at 1733750554432Finishing writing output for hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223/wal-2 so closing down at 1733750554453 (+21 ms)Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/recovered.edits/0000000000000000002-wal-2.temp at 1733750554466 (+13 ms)3 split writer threads finished at 1733750554466Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733750554492 (+26 ms)Rename recovered edits hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/recovered.edits/0000000000000000002 at 1733750554497 (+5 ms)Processed 1 edits across 1 Regions in 51 ms; skipped=0; WAL=hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1733750554497 2024-12-09T13:22:34,520 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223/wal-1, size=369 (369bytes) 2024-12-09T13:22:34,520 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223/wal-1 2024-12-09T13:22:34,521 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223/wal-1 after 1ms 2024-12-09T13:22:34,526 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223/wal-1 took 6ms 2024-12-09T13:22:34,539 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223/wal-1 so closing down 2024-12-09T13:22:34,539 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T13:22:34,545 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-09T13:22:34,547 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/recovered.edits/0000000000000000001-wal-1.temp 2024-12-09T13:22:34,547 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T13:22:34,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741850_1026 (size=369) 2024-12-09T13:22:34,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741850_1026 (size=369) 2024-12-09T13:22:34,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741850_1026 (size=369) 2024-12-09T13:22:34,579 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-09T13:22:34,587 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/recovered.edits/0000000000000000002, length=231 2024-12-09T13:22:34,590 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/recovered.edits/0000000000000000002 2024-12-09T13:22:34,590 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 57 ms; skipped=0; WAL=hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-12-09T13:22:34,590 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223/wal-1, journal: Splitting hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223/wal-1, size=369 (369bytes) at 1733750554520Finishing writing output for hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223/wal-1 so closing down at 1733750554539 (+19 ms)Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/recovered.edits/0000000000000000001-wal-1.temp at 1733750554547 (+8 ms)3 split writer threads finished at 1733750554548 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733750554579 (+31 ms)Rename recovered edits hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/recovered.edits/0000000000000000002 at 1733750554590 (+11 ms)Processed 2 edits across 1 Regions in 57 ms; skipped=0; WAL=hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1733750554590 2024-12-09T13:22:34,590 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T13:22:34,595 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:22:34,618 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223/wal.1733750554599, exclude list is [], retry=0 2024-12-09T13:22:34,622 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:22:34,622 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:22:34,623 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:22:34,627 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223/wal.1733750554599 2024-12-09T13:22:34,630 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34135:34135),(127.0.0.1/127.0.0.1:34323:34323)] 2024-12-09T13:22:34,631 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 44a05a8bda3ea46f351fd358f6b43061, NAME => 'testReplayEditsWrittenIntoWAL,,1733750554225.44a05a8bda3ea46f351fd358f6b43061.', STARTKEY => '', ENDKEY => ''} 2024-12-09T13:22:34,631 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733750554225.44a05a8bda3ea46f351fd358f6b43061.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:34,631 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 44a05a8bda3ea46f351fd358f6b43061 2024-12-09T13:22:34,631 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 44a05a8bda3ea46f351fd358f6b43061 2024-12-09T13:22:34,634 INFO [StoreOpener-44a05a8bda3ea46f351fd358f6b43061-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 44a05a8bda3ea46f351fd358f6b43061 2024-12-09T13:22:34,640 INFO [StoreOpener-44a05a8bda3ea46f351fd358f6b43061-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44a05a8bda3ea46f351fd358f6b43061 columnFamilyName a 2024-12-09T13:22:34,640 DEBUG [StoreOpener-44a05a8bda3ea46f351fd358f6b43061-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:34,643 INFO [StoreOpener-44a05a8bda3ea46f351fd358f6b43061-1 {}] regionserver.HStore(327): Store=44a05a8bda3ea46f351fd358f6b43061/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:34,643 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 44a05a8bda3ea46f351fd358f6b43061 2024-12-09T13:22:34,644 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061 2024-12-09T13:22:34,654 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061 2024-12-09T13:22:34,655 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/recovered.edits/0000000000000000002 2024-12-09T13:22:34,659 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/recovered.edits/0000000000000000002 2024-12-09T13:22:34,659 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 44a05a8bda3ea46f351fd358f6b43061 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-09T13:22:34,691 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/.tmp/a/0a8dbe07c36e4386bcf11d1692b4b7c8 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733750554358/Put/seqid=0 2024-12-09T13:22:34,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741852_1028 (size=5170) 2024-12-09T13:22:34,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741852_1028 (size=5170) 2024-12-09T13:22:34,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741852_1028 (size=5170) 2024-12-09T13:22:34,708 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/.tmp/a/0a8dbe07c36e4386bcf11d1692b4b7c8 2024-12-09T13:22:34,727 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/.tmp/a/0a8dbe07c36e4386bcf11d1692b4b7c8 as hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/a/0a8dbe07c36e4386bcf11d1692b4b7c8 2024-12-09T13:22:34,736 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/a/0a8dbe07c36e4386bcf11d1692b4b7c8, entries=2, sequenceid=2, filesize=5.0 K 2024-12-09T13:22:34,736 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 44a05a8bda3ea46f351fd358f6b43061 in 77ms, sequenceid=2, compaction requested=false; wal=null 2024-12-09T13:22:34,737 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/recovered.edits/0000000000000000002 2024-12-09T13:22:34,741 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 44a05a8bda3ea46f351fd358f6b43061 2024-12-09T13:22:34,741 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 44a05a8bda3ea46f351fd358f6b43061 2024-12-09T13:22:34,744 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 44a05a8bda3ea46f351fd358f6b43061 2024-12-09T13:22:34,747 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/44a05a8bda3ea46f351fd358f6b43061/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-09T13:22:34,749 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 44a05a8bda3ea46f351fd358f6b43061; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64334051, jitterRate=-0.041347935795784}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T13:22:34,749 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 44a05a8bda3ea46f351fd358f6b43061: Writing region info on filesystem at 1733750554631Initializing all the Stores at 1733750554633 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750554633Obtaining lock to block concurrent updates at 1733750554659 (+26 ms)Preparing flush snapshotting stores in 44a05a8bda3ea46f351fd358f6b43061 at 1733750554659Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733750554225.44a05a8bda3ea46f351fd358f6b43061., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733750554659Flushing stores of testReplayEditsWrittenIntoWAL,,1733750554225.44a05a8bda3ea46f351fd358f6b43061. at 1733750554659Flushing 44a05a8bda3ea46f351fd358f6b43061/a: creating writer at 1733750554659Flushing 44a05a8bda3ea46f351fd358f6b43061/a: appending metadata at 1733750554689 (+30 ms)Flushing 44a05a8bda3ea46f351fd358f6b43061/a: closing flushed file at 1733750554689Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@749fa156: reopening flushed file at 1733750554722 (+33 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 44a05a8bda3ea46f351fd358f6b43061 in 77ms, sequenceid=2, compaction requested=false; wal=null at 1733750554736 (+14 ms)Cleaning up temporary data from old regions at 1733750554741 (+5 ms)Region opened successfully at 1733750554749 (+8 ms) 2024-12-09T13:22:34,773 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=380 (was 370) Potentially hanging thread: PacketResponder: BP-1859740653-172.17.0.3-1733750541580:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:54288 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:60570 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1859740653-172.17.0.3-1733750541580:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:54392 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1859740653-172.17.0.3-1733750541580:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:54504 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:54400 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:60758 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=707 (was 707), ProcessCount=11 (was 11), AvailableMemoryMB=4019 (was 4026) 2024-12-09T13:22:34,787 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=380, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=707, ProcessCount=11, AvailableMemoryMB=4019 2024-12-09T13:22:34,807 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:22:34,809 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:22:34,810 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T13:22:34,822 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-44045720, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/hregion-44045720, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:22:34,843 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-44045720/hregion-44045720.1733750554822, exclude list is [], retry=0 2024-12-09T13:22:34,848 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:22:34,849 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:22:34,850 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:22:34,859 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-44045720/hregion-44045720.1733750554822 2024-12-09T13:22:34,862 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34135:34135),(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34323:34323)] 2024-12-09T13:22:34,862 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => bfaee1ae00378a2ccd537a17df37f694, NAME => 'testReplayEditsWrittenIntoWAL,,1733750554808.bfaee1ae00378a2ccd537a17df37f694.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41625/hbase 2024-12-09T13:22:34,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741854_1030 (size=64) 2024-12-09T13:22:34,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741854_1030 (size=64) 2024-12-09T13:22:34,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741854_1030 (size=64) 2024-12-09T13:22:34,893 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733750554808.bfaee1ae00378a2ccd537a17df37f694.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:34,895 INFO [StoreOpener-bfaee1ae00378a2ccd537a17df37f694-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bfaee1ae00378a2ccd537a17df37f694 2024-12-09T13:22:34,904 INFO [StoreOpener-bfaee1ae00378a2ccd537a17df37f694-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfaee1ae00378a2ccd537a17df37f694 columnFamilyName a 2024-12-09T13:22:34,905 DEBUG [StoreOpener-bfaee1ae00378a2ccd537a17df37f694-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:34,905 INFO [StoreOpener-bfaee1ae00378a2ccd537a17df37f694-1 {}] regionserver.HStore(327): Store=bfaee1ae00378a2ccd537a17df37f694/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:34,906 INFO [StoreOpener-bfaee1ae00378a2ccd537a17df37f694-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bfaee1ae00378a2ccd537a17df37f694 2024-12-09T13:22:34,907 INFO [StoreOpener-bfaee1ae00378a2ccd537a17df37f694-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfaee1ae00378a2ccd537a17df37f694 columnFamilyName b 2024-12-09T13:22:34,908 DEBUG [StoreOpener-bfaee1ae00378a2ccd537a17df37f694-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:34,908 INFO [StoreOpener-bfaee1ae00378a2ccd537a17df37f694-1 {}] regionserver.HStore(327): Store=bfaee1ae00378a2ccd537a17df37f694/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:34,909 INFO [StoreOpener-bfaee1ae00378a2ccd537a17df37f694-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bfaee1ae00378a2ccd537a17df37f694 2024-12-09T13:22:34,911 INFO [StoreOpener-bfaee1ae00378a2ccd537a17df37f694-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfaee1ae00378a2ccd537a17df37f694 columnFamilyName c 2024-12-09T13:22:34,911 DEBUG [StoreOpener-bfaee1ae00378a2ccd537a17df37f694-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:34,912 INFO [StoreOpener-bfaee1ae00378a2ccd537a17df37f694-1 {}] regionserver.HStore(327): Store=bfaee1ae00378a2ccd537a17df37f694/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:34,912 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bfaee1ae00378a2ccd537a17df37f694 2024-12-09T13:22:34,913 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694 2024-12-09T13:22:34,913 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694 2024-12-09T13:22:34,915 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bfaee1ae00378a2ccd537a17df37f694 2024-12-09T13:22:34,915 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bfaee1ae00378a2ccd537a17df37f694 2024-12-09T13:22:34,916 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T13:22:34,917 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bfaee1ae00378a2ccd537a17df37f694 2024-12-09T13:22:34,920 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T13:22:34,921 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bfaee1ae00378a2ccd537a17df37f694; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74548023, jitterRate=0.11085210740566254}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T13:22:34,921 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bfaee1ae00378a2ccd537a17df37f694: Writing region info on filesystem at 1733750554893Initializing all the Stores at 1733750554894 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750554894Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750554895 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750554895Cleaning up temporary data from old regions at 1733750554915 (+20 ms)Region opened successfully at 1733750554921 (+6 ms) 2024-12-09T13:22:34,922 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bfaee1ae00378a2ccd537a17df37f694, disabling compactions & flushes 2024-12-09T13:22:34,922 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733750554808.bfaee1ae00378a2ccd537a17df37f694. 2024-12-09T13:22:34,922 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733750554808.bfaee1ae00378a2ccd537a17df37f694. 2024-12-09T13:22:34,922 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733750554808.bfaee1ae00378a2ccd537a17df37f694. after waiting 0 ms 2024-12-09T13:22:34,922 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733750554808.bfaee1ae00378a2ccd537a17df37f694. 2024-12-09T13:22:34,923 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733750554808.bfaee1ae00378a2ccd537a17df37f694. 2024-12-09T13:22:34,923 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bfaee1ae00378a2ccd537a17df37f694: Waiting for close lock at 1733750554922Disabling compacts and flushes for region at 1733750554922Disabling writes for close at 1733750554922Writing region close event to WAL at 1733750554922Closed at 1733750554922 2024-12-09T13:22:34,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741853_1029 (size=93) 2024-12-09T13:22:34,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741853_1029 (size=93) 2024-12-09T13:22:34,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741853_1029 (size=93) 2024-12-09T13:22:34,945 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T13:22:34,945 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-44045720:(num 1733750554822) 2024-12-09T13:22:34,945 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T13:22:34,947 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733750554806, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:22:34,966 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733750554806/wal.1733750554948, exclude list is [], retry=0 2024-12-09T13:22:34,969 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:22:34,970 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:22:34,970 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:22:34,976 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733750554806/wal.1733750554948 2024-12-09T13:22:34,976 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34135:34135),(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34323:34323)] 2024-12-09T13:22:35,247 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733750554806/wal.1733750554948, size=0 (0bytes) 2024-12-09T13:22:35,247 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733750554806/wal.1733750554948 might be still open, length is 0 2024-12-09T13:22:35,247 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733750554806/wal.1733750554948 2024-12-09T13:22:35,248 WARN [IPC Server handler 1 on default port 41625 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733750554806/wal.1733750554948 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-09T13:22:35,249 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733750554806/wal.1733750554948 after 2ms 2024-12-09T13:22:37,292 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:54428 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37811:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54428 dst: /127.0.0.1:37811 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37811 remote=/127.0.0.1:54428]. Total timeout mills is 60000, 57904 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:22:37,293 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:60778 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:40963:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60778 dst: /127.0.0.1:40963 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:22:37,295 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:54540 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41523:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54540 dst: /127.0.0.1:41523 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:22:37,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741855_1032 (size=470675) 2024-12-09T13:22:37,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741855_1032 (size=470675) 2024-12-09T13:22:37,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741855_1032 (size=470675) 2024-12-09T13:22:37,343 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-09T13:22:37,975 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T13:22:38,049 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-09T13:22:38,050 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-09T13:22:38,052 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T13:22:38,052 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-09T13:22:38,053 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-09T13:22:38,053 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-09T13:22:38,054 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-09T13:22:38,054 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-09T13:22:39,250 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733750554806/wal.1733750554948 after 4003ms 2024-12-09T13:22:39,264 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733750554806/wal.1733750554948 took 4017ms 2024-12-09T13:22:39,279 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733750554948.temp 2024-12-09T13:22:39,286 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/recovered.edits/0000000000000000001-wal.1733750554948.temp 2024-12-09T13:22:39,408 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733750554806/wal.1733750554948; continuing. 2024-12-09T13:22:39,408 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733750554806/wal.1733750554948 so closing down 2024-12-09T13:22:39,408 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T13:22:39,409 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T13:22:39,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741856_1033 (size=470683) 2024-12-09T13:22:39,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741856_1033 (size=470683) 2024-12-09T13:22:39,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741856_1033 (size=470683) 2024-12-09T13:22:39,439 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/recovered.edits/0000000000000000001-wal.1733750554948.temp (wrote 3002 edits, skipped 0 edits in 31 ms) 2024-12-09T13:22:39,466 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/recovered.edits/0000000000000000001-wal.1733750554948.temp to hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/recovered.edits/0000000000000003002 2024-12-09T13:22:39,467 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 198 ms; skipped=0; WAL=hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733750554806/wal.1733750554948, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T13:22:39,471 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733750554806/wal.1733750554948, journal: Splitting hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733750554806/wal.1733750554948, size=0 (0bytes) at 1733750555247Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/recovered.edits/0000000000000000001-wal.1733750554948.temp at 1733750559287 (+4040 ms)Split 1024 edits, skipped 0 edits. at 1733750559334 (+47 ms)Split 2048 edits, skipped 0 edits. at 1733750559392 (+58 ms)Finishing writing output for hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733750554806/wal.1733750554948 so closing down at 1733750559408 (+16 ms)3 split writer threads finished at 1733750559409 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/recovered.edits/0000000000000000001-wal.1733750554948.temp (wrote 3002 edits, skipped 0 edits in 31 ms) at 1733750559439 (+30 ms)Rename recovered edits hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/recovered.edits/0000000000000000001-wal.1733750554948.temp to hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/recovered.edits/0000000000000003002 at 1733750559466 (+27 ms)Processed 3002 edits across 1 Regions in 198 ms; skipped=0; WAL=hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733750554806/wal.1733750554948, size=0, length=0, corrupted=false, cancelled=false at 1733750559467 (+1 ms) 2024-12-09T13:22:39,491 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733750554806/wal.1733750554948 to hdfs://localhost:41625/hbase/oldWALs/wal.1733750554948 2024-12-09T13:22:39,494 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/recovered.edits/0000000000000003002 2024-12-09T13:22:39,494 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T13:22:39,499 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733750554806, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:22:39,521 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733750554806/wal.1733750559500, exclude list is [], retry=0 2024-12-09T13:22:39,529 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:22:39,529 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:22:39,530 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:22:39,551 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733750554806/wal.1733750559500 2024-12-09T13:22:39,552 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34135:34135),(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34323:34323)] 2024-12-09T13:22:39,552 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733750554808.bfaee1ae00378a2ccd537a17df37f694.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:39,572 INFO [StoreOpener-bfaee1ae00378a2ccd537a17df37f694-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bfaee1ae00378a2ccd537a17df37f694 2024-12-09T13:22:39,584 INFO [StoreOpener-bfaee1ae00378a2ccd537a17df37f694-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfaee1ae00378a2ccd537a17df37f694 columnFamilyName a 2024-12-09T13:22:39,585 DEBUG [StoreOpener-bfaee1ae00378a2ccd537a17df37f694-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:39,586 INFO [StoreOpener-bfaee1ae00378a2ccd537a17df37f694-1 {}] regionserver.HStore(327): Store=bfaee1ae00378a2ccd537a17df37f694/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:39,586 INFO [StoreOpener-bfaee1ae00378a2ccd537a17df37f694-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bfaee1ae00378a2ccd537a17df37f694 2024-12-09T13:22:39,588 INFO [StoreOpener-bfaee1ae00378a2ccd537a17df37f694-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfaee1ae00378a2ccd537a17df37f694 columnFamilyName b 2024-12-09T13:22:39,588 DEBUG [StoreOpener-bfaee1ae00378a2ccd537a17df37f694-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:39,589 INFO [StoreOpener-bfaee1ae00378a2ccd537a17df37f694-1 {}] regionserver.HStore(327): Store=bfaee1ae00378a2ccd537a17df37f694/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:39,589 INFO [StoreOpener-bfaee1ae00378a2ccd537a17df37f694-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bfaee1ae00378a2ccd537a17df37f694 2024-12-09T13:22:39,590 INFO [StoreOpener-bfaee1ae00378a2ccd537a17df37f694-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfaee1ae00378a2ccd537a17df37f694 columnFamilyName c 2024-12-09T13:22:39,590 DEBUG [StoreOpener-bfaee1ae00378a2ccd537a17df37f694-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:39,591 INFO [StoreOpener-bfaee1ae00378a2ccd537a17df37f694-1 {}] regionserver.HStore(327): Store=bfaee1ae00378a2ccd537a17df37f694/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:39,591 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bfaee1ae00378a2ccd537a17df37f694 2024-12-09T13:22:39,592 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694 2024-12-09T13:22:39,611 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694 2024-12-09T13:22:39,613 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/recovered.edits/0000000000000003002 2024-12-09T13:22:39,665 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T13:22:40,195 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bfaee1ae00378a2ccd537a17df37f694 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-09T13:22:40,249 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/.tmp/a/d8e9a85718124214a6ea84455c155363 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733750554985/Put/seqid=0 2024-12-09T13:22:40,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741858_1035 (size=50463) 2024-12-09T13:22:40,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741858_1035 (size=50463) 2024-12-09T13:22:40,279 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/.tmp/a/d8e9a85718124214a6ea84455c155363 2024-12-09T13:22:40,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741858_1035 (size=50463) 2024-12-09T13:22:40,301 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/.tmp/a/d8e9a85718124214a6ea84455c155363 as hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/a/d8e9a85718124214a6ea84455c155363 2024-12-09T13:22:40,309 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/a/d8e9a85718124214a6ea84455c155363, entries=754, sequenceid=754, filesize=49.3 K 2024-12-09T13:22:40,309 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for bfaee1ae00378a2ccd537a17df37f694 in 114ms, sequenceid=754, compaction requested=false; wal=null 2024-12-09T13:22:40,331 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T13:22:40,332 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bfaee1ae00378a2ccd537a17df37f694 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-09T13:22:40,342 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/.tmp/a/6686b7b954bd41e3b0a1087ed4623b60 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733750555044/Put/seqid=0 2024-12-09T13:22:40,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741859_1036 (size=20072) 2024-12-09T13:22:40,430 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/.tmp/a/6686b7b954bd41e3b0a1087ed4623b60 2024-12-09T13:22:40,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741859_1036 (size=20072) 2024-12-09T13:22:40,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741859_1036 (size=20072) 2024-12-09T13:22:40,493 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/.tmp/b/3af7a846cf934d05964a897c58b6bcb4 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733750555069/Put/seqid=0 2024-12-09T13:22:40,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741860_1037 (size=35835) 2024-12-09T13:22:40,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741860_1037 (size=35835) 2024-12-09T13:22:40,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741860_1037 (size=35835) 2024-12-09T13:22:40,936 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/.tmp/b/3af7a846cf934d05964a897c58b6bcb4 2024-12-09T13:22:40,954 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/.tmp/a/6686b7b954bd41e3b0a1087ed4623b60 as hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/a/6686b7b954bd41e3b0a1087ed4623b60 2024-12-09T13:22:40,984 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/a/6686b7b954bd41e3b0a1087ed4623b60, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-09T13:22:40,986 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/.tmp/b/3af7a846cf934d05964a897c58b6bcb4 as hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/b/3af7a846cf934d05964a897c58b6bcb4 2024-12-09T13:22:41,008 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/b/3af7a846cf934d05964a897c58b6bcb4, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-09T13:22:41,009 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for bfaee1ae00378a2ccd537a17df37f694 in 677ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-09T13:22:41,023 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T13:22:41,024 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bfaee1ae00378a2ccd537a17df37f694 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-09T13:22:41,053 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/.tmp/b/2f28b5ce93fe4dfda90189a59fa521a6 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733750555093/Put/seqid=0 2024-12-09T13:22:41,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741861_1038 (size=35082) 2024-12-09T13:22:41,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741861_1038 (size=35082) 2024-12-09T13:22:41,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741861_1038 (size=35082) 2024-12-09T13:22:41,111 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/.tmp/b/2f28b5ce93fe4dfda90189a59fa521a6 2024-12-09T13:22:41,179 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/.tmp/c/748bc9058a174247be481569d8709931 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733750555131/Put/seqid=0 2024-12-09T13:22:41,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741862_1039 (size=20825) 2024-12-09T13:22:41,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741862_1039 (size=20825) 2024-12-09T13:22:41,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741862_1039 (size=20825) 2024-12-09T13:22:41,623 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/.tmp/c/748bc9058a174247be481569d8709931 2024-12-09T13:22:41,633 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/.tmp/b/2f28b5ce93fe4dfda90189a59fa521a6 as hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/b/2f28b5ce93fe4dfda90189a59fa521a6 2024-12-09T13:22:41,643 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/b/2f28b5ce93fe4dfda90189a59fa521a6, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-09T13:22:41,645 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/.tmp/c/748bc9058a174247be481569d8709931 as hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/c/748bc9058a174247be481569d8709931 2024-12-09T13:22:41,658 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/c/748bc9058a174247be481569d8709931, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-09T13:22:41,659 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for bfaee1ae00378a2ccd537a17df37f694 in 636ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-09T13:22:41,671 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733750555192/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733750554808.bfaee1ae00378a2ccd537a17df37f694. 2024-12-09T13:22:41,675 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/recovered.edits/0000000000000003002 2024-12-09T13:22:41,676 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T13:22:41,676 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bfaee1ae00378a2ccd537a17df37f694 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-09T13:22:41,685 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/.tmp/c/2069b837e3694bc1b3825a6dc7984d05 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733750555139/Put/seqid=0 2024-12-09T13:22:41,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741863_1040 (size=50301) 2024-12-09T13:22:41,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741863_1040 (size=50301) 2024-12-09T13:22:41,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741863_1040 (size=50301) 2024-12-09T13:22:41,764 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/.tmp/c/2069b837e3694bc1b3825a6dc7984d05 2024-12-09T13:22:41,788 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2069b837e3694bc1b3825a6dc7984d05 2024-12-09T13:22:41,802 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/.tmp/c/2069b837e3694bc1b3825a6dc7984d05 as hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/c/2069b837e3694bc1b3825a6dc7984d05 2024-12-09T13:22:41,835 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2069b837e3694bc1b3825a6dc7984d05 2024-12-09T13:22:41,836 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/c/2069b837e3694bc1b3825a6dc7984d05, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-09T13:22:41,838 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for bfaee1ae00378a2ccd537a17df37f694 in 161ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-09T13:22:41,840 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/recovered.edits/0000000000000003002 2024-12-09T13:22:41,842 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bfaee1ae00378a2ccd537a17df37f694 2024-12-09T13:22:41,842 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bfaee1ae00378a2ccd537a17df37f694 2024-12-09T13:22:41,846 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T13:22:41,861 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bfaee1ae00378a2ccd537a17df37f694 2024-12-09T13:22:41,871 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenIntoWAL/bfaee1ae00378a2ccd537a17df37f694/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-09T13:22:41,886 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bfaee1ae00378a2ccd537a17df37f694; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65795901, jitterRate=-0.0195646733045578}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T13:22:41,887 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bfaee1ae00378a2ccd537a17df37f694: Writing region info on filesystem at 1733750559552Initializing all the Stores at 1733750559566 (+14 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750559566Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750559569 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750559569Cleaning up temporary data from old regions at 1733750561842 (+2273 ms)Region opened successfully at 1733750561886 (+44 ms) 2024-12-09T13:22:42,026 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bfaee1ae00378a2ccd537a17df37f694, disabling compactions & flushes 2024-12-09T13:22:42,026 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733750554808.bfaee1ae00378a2ccd537a17df37f694. 2024-12-09T13:22:42,026 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733750554808.bfaee1ae00378a2ccd537a17df37f694. 2024-12-09T13:22:42,026 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733750554808.bfaee1ae00378a2ccd537a17df37f694. after waiting 0 ms 2024-12-09T13:22:42,026 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733750554808.bfaee1ae00378a2ccd537a17df37f694. 2024-12-09T13:22:42,054 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733750554808.bfaee1ae00378a2ccd537a17df37f694. 2024-12-09T13:22:42,054 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bfaee1ae00378a2ccd537a17df37f694: Waiting for close lock at 1733750562026Disabling compacts and flushes for region at 1733750562026Disabling writes for close at 1733750562026Writing region close event to WAL at 1733750562054 (+28 ms)Closed at 1733750562054 2024-12-09T13:22:42,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741857_1034 (size=93) 2024-12-09T13:22:42,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741857_1034 (size=93) 2024-12-09T13:22:42,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741857_1034 (size=93) 2024-12-09T13:22:42,084 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T13:22:42,084 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733750559500) 2024-12-09T13:22:42,112 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=394 (was 380) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43635 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/cluster_b6088222-ea39-2e85-344a-dadc86871089/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/cluster_b6088222-ea39-2e85-344a-dadc86871089/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:41625 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/cluster_b6088222-ea39-2e85-344a-dadc86871089/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_197754876_22 at /127.0.0.1:54122 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/cluster_b6088222-ea39-2e85-344a-dadc86871089/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34651 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (587680655) connection to localhost/127.0.0.1:43635 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_197754876_22 at /127.0.0.1:53308 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_197754876_22 at /127.0.0.1:49308 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (587680655) connection to localhost/127.0.0.1:41625 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (587680655) connection to localhost/127.0.0.1:34651 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_197754876_22 at /127.0.0.1:54068 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=861 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=683 (was 707), ProcessCount=11 (was 11), AvailableMemoryMB=3838 (was 4019) 2024-12-09T13:22:42,134 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#test2727 Thread=394, OpenFileDescriptor=861, MaxFileDescriptor=1048576, SystemLoadAverage=683, ProcessCount=11, AvailableMemoryMB=3837 2024-12-09T13:22:42,173 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:22:42,177 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:22:42,178 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T13:22:42,182 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-57951615, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/hregion-57951615, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:22:42,198 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-57951615/hregion-57951615.1733750562182, exclude list is [], retry=0 2024-12-09T13:22:42,202 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:22:42,202 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:22:42,203 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:22:42,205 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-57951615/hregion-57951615.1733750562182 2024-12-09T13:22:42,206 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34323:34323),(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34135:34135)] 2024-12-09T13:22:42,206 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 19fccbe3064ee0df9e08299c115dd3c2, NAME => 'test2727,,1733750562175.19fccbe3064ee0df9e08299c115dd3c2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41625/hbase 2024-12-09T13:22:42,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741865_1042 (size=43) 2024-12-09T13:22:42,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741865_1042 (size=43) 2024-12-09T13:22:42,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741865_1042 (size=43) 2024-12-09T13:22:42,225 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733750562175.19fccbe3064ee0df9e08299c115dd3c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:42,228 INFO [StoreOpener-19fccbe3064ee0df9e08299c115dd3c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 19fccbe3064ee0df9e08299c115dd3c2 2024-12-09T13:22:42,230 INFO [StoreOpener-19fccbe3064ee0df9e08299c115dd3c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 19fccbe3064ee0df9e08299c115dd3c2 columnFamilyName a 2024-12-09T13:22:42,230 DEBUG [StoreOpener-19fccbe3064ee0df9e08299c115dd3c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:42,231 INFO [StoreOpener-19fccbe3064ee0df9e08299c115dd3c2-1 {}] regionserver.HStore(327): Store=19fccbe3064ee0df9e08299c115dd3c2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:42,231 INFO [StoreOpener-19fccbe3064ee0df9e08299c115dd3c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 19fccbe3064ee0df9e08299c115dd3c2 2024-12-09T13:22:42,233 INFO [StoreOpener-19fccbe3064ee0df9e08299c115dd3c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 19fccbe3064ee0df9e08299c115dd3c2 columnFamilyName b 2024-12-09T13:22:42,233 DEBUG [StoreOpener-19fccbe3064ee0df9e08299c115dd3c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:42,233 INFO [StoreOpener-19fccbe3064ee0df9e08299c115dd3c2-1 {}] regionserver.HStore(327): Store=19fccbe3064ee0df9e08299c115dd3c2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:42,234 INFO [StoreOpener-19fccbe3064ee0df9e08299c115dd3c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 19fccbe3064ee0df9e08299c115dd3c2 2024-12-09T13:22:42,235 INFO [StoreOpener-19fccbe3064ee0df9e08299c115dd3c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 19fccbe3064ee0df9e08299c115dd3c2 columnFamilyName c 2024-12-09T13:22:42,235 DEBUG [StoreOpener-19fccbe3064ee0df9e08299c115dd3c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:42,236 INFO [StoreOpener-19fccbe3064ee0df9e08299c115dd3c2-1 {}] regionserver.HStore(327): Store=19fccbe3064ee0df9e08299c115dd3c2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:42,236 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 19fccbe3064ee0df9e08299c115dd3c2 2024-12-09T13:22:42,237 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2 2024-12-09T13:22:42,237 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2 2024-12-09T13:22:42,239 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 19fccbe3064ee0df9e08299c115dd3c2 2024-12-09T13:22:42,239 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 19fccbe3064ee0df9e08299c115dd3c2 2024-12-09T13:22:42,240 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T13:22:42,241 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 19fccbe3064ee0df9e08299c115dd3c2 2024-12-09T13:22:42,246 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T13:22:42,246 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 19fccbe3064ee0df9e08299c115dd3c2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67556282, jitterRate=0.006667047739028931}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T13:22:42,248 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 19fccbe3064ee0df9e08299c115dd3c2: Writing region info on filesystem at 1733750562225Initializing all the Stores at 1733750562227 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750562227Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750562228 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750562228Cleaning up temporary data from old regions at 1733750562239 (+11 ms)Region opened successfully at 1733750562248 (+9 ms) 2024-12-09T13:22:42,248 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 19fccbe3064ee0df9e08299c115dd3c2, disabling compactions & flushes 2024-12-09T13:22:42,248 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733750562175.19fccbe3064ee0df9e08299c115dd3c2. 2024-12-09T13:22:42,248 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733750562175.19fccbe3064ee0df9e08299c115dd3c2. 2024-12-09T13:22:42,248 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733750562175.19fccbe3064ee0df9e08299c115dd3c2. after waiting 0 ms 2024-12-09T13:22:42,248 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733750562175.19fccbe3064ee0df9e08299c115dd3c2. 2024-12-09T13:22:42,251 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733750562175.19fccbe3064ee0df9e08299c115dd3c2. 2024-12-09T13:22:42,251 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 19fccbe3064ee0df9e08299c115dd3c2: Waiting for close lock at 1733750562248Disabling compacts and flushes for region at 1733750562248Disabling writes for close at 1733750562248Writing region close event to WAL at 1733750562250 (+2 ms)Closed at 1733750562251 (+1 ms) 2024-12-09T13:22:42,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741864_1041 (size=93) 2024-12-09T13:22:42,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741864_1041 (size=93) 2024-12-09T13:22:42,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741864_1041 (size=93) 2024-12-09T13:22:42,262 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T13:22:42,262 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-57951615:(num 1733750562182) 2024-12-09T13:22:42,263 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T13:22:42,265 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/test2727-manual,16010,1733750562173, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:22:42,282 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562266, exclude list is [], retry=0 2024-12-09T13:22:42,286 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:22:42,287 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:22:42,287 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:22:42,307 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562266 2024-12-09T13:22:42,327 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34323:34323),(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34135:34135)] 2024-12-09T13:22:42,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741866_1043 (size=344306) 2024-12-09T13:22:42,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741866_1043 (size=344306) 2024-12-09T13:22:42,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741866_1043 (size=344306) 2024-12-09T13:22:42,659 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41625/hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562266, size=336.2 K (344306bytes) 2024-12-09T13:22:42,659 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41625/hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562266 2024-12-09T13:22:42,660 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41625/hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562266 after 1ms 2024-12-09T13:22:42,672 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41625/hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562266 took 13ms 2024-12-09T13:22:42,757 INFO [Time-limited test-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733750562266.temp 2024-12-09T13:22:42,767 INFO [Time-limited test-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/0000000000000000001-wal.1733750562266.temp 2024-12-09T13:22:42,789 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41625/hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562266 so closing down 2024-12-09T13:22:42,789 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T13:22:42,790 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T13:22:42,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741867_1044 (size=344306) 2024-12-09T13:22:42,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741867_1044 (size=344306) 2024-12-09T13:22:42,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741867_1044 (size=344306) 2024-12-09T13:22:42,823 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/0000000000000000001-wal.1733750562266.temp (wrote 3000 edits, skipped 0 edits in 18 ms) 2024-12-09T13:22:42,847 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/0000000000000000001-wal.1733750562266.temp to hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/0000000000000003000 2024-12-09T13:22:42,847 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 161 ms; skipped=0; WAL=hdfs://localhost:41625/hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562266, size=336.2 K, length=344306, corrupted=false, cancelled=false 2024-12-09T13:22:42,847 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41625/hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562266, journal: Splitting hdfs://localhost:41625/hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562266, size=336.2 K (344306bytes) at 1733750562659Split 1024 edits, skipped 0 edits. at 1733750562754 (+95 ms)Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/0000000000000000001-wal.1733750562266.temp at 1733750562767 (+13 ms)Split 2048 edits, skipped 0 edits. at 1733750562769 (+2 ms)Finishing writing output for hdfs://localhost:41625/hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562266 so closing down at 1733750562789 (+20 ms)3 split writer threads finished at 1733750562790 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/0000000000000000001-wal.1733750562266.temp (wrote 3000 edits, skipped 0 edits in 18 ms) at 1733750562823 (+33 ms)Rename recovered edits hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/0000000000000000001-wal.1733750562266.temp to hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/0000000000000003000 at 1733750562847 (+24 ms)Processed 3000 edits across 1 Regions in 161 ms; skipped=0; WAL=hdfs://localhost:41625/hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562266, size=336.2 K, length=344306, corrupted=false, cancelled=false at 1733750562847 2024-12-09T13:22:42,870 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41625/hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562266 to hdfs://localhost:41625/hbase/oldWALs/wal.1733750562266 2024-12-09T13:22:42,871 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/0000000000000003000 2024-12-09T13:22:42,872 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T13:22:42,874 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/test2727-manual,16010,1733750562173, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:22:42,896 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562875, exclude list is [], retry=0 2024-12-09T13:22:42,905 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:22:42,905 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:22:42,906 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:22:42,912 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562875 2024-12-09T13:22:42,913 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34323:34323),(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34135:34135)] 2024-12-09T13:22:43,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741868_1045 (size=344433) 2024-12-09T13:22:43,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741868_1045 (size=344433) 2024-12-09T13:22:43,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741868_1045 (size=344433) 2024-12-09T13:22:43,132 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562875 not finished, retry = 0 2024-12-09T13:22:43,253 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41625/hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562875, size=336.4 K (344433bytes) 2024-12-09T13:22:43,253 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41625/hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562875 2024-12-09T13:22:43,254 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41625/hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562875 after 1ms 2024-12-09T13:22:43,263 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41625/hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562875 took 10ms 2024-12-09T13:22:43,277 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733750562875.temp 2024-12-09T13:22:43,279 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/0000000000000003001-wal.1733750562875.temp 2024-12-09T13:22:43,287 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41625/hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562875 so closing down 2024-12-09T13:22:43,287 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T13:22:43,288 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T13:22:43,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741869_1046 (size=344433) 2024-12-09T13:22:43,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741869_1046 (size=344433) 2024-12-09T13:22:43,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741869_1046 (size=344433) 2024-12-09T13:22:43,324 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/0000000000000003001-wal.1733750562875.temp (wrote 3000 edits, skipped 0 edits in 6 ms) 2024-12-09T13:22:43,334 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/0000000000000003001-wal.1733750562875.temp to hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/0000000000000006000 2024-12-09T13:22:43,335 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 70 ms; skipped=0; WAL=hdfs://localhost:41625/hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562875, size=336.4 K, length=344433, corrupted=false, cancelled=false 2024-12-09T13:22:43,335 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41625/hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562875, journal: Splitting hdfs://localhost:41625/hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562875, size=336.4 K (344433bytes) at 1733750563253Split 1024 edits, skipped 0 edits. at 1733750563275 (+22 ms)Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/0000000000000003001-wal.1733750562875.temp at 1733750563279 (+4 ms)Split 2048 edits, skipped 0 edits. at 1733750563282 (+3 ms)Finishing writing output for hdfs://localhost:41625/hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562875 so closing down at 1733750563287 (+5 ms)3 split writer threads finished at 1733750563288 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/0000000000000003001-wal.1733750562875.temp (wrote 3000 edits, skipped 0 edits in 6 ms) at 1733750563324 (+36 ms)Rename recovered edits hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/0000000000000003001-wal.1733750562875.temp to hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/0000000000000006000 at 1733750563334 (+10 ms)Processed 3000 edits across 1 Regions in 70 ms; skipped=0; WAL=hdfs://localhost:41625/hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562875, size=336.4 K, length=344433, corrupted=false, cancelled=false at 1733750563335 (+1 ms) 2024-12-09T13:22:43,342 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41625/hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750562875 to hdfs://localhost:41625/hbase/oldWALs/wal.1733750562875 2024-12-09T13:22:43,346 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/0000000000000006000 2024-12-09T13:22:43,347 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T13:22:43,353 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/test2727-manual,16010,1733750562173, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:22:43,375 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750563353, exclude list is [], retry=0 2024-12-09T13:22:43,379 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:22:43,379 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:22:43,379 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:22:43,386 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733750562173/wal.1733750563353 2024-12-09T13:22:43,386 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34135:34135),(127.0.0.1/127.0.0.1:34323:34323),(127.0.0.1/127.0.0.1:44161:44161)] 2024-12-09T13:22:43,387 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 19fccbe3064ee0df9e08299c115dd3c2, NAME => 'test2727,,1733750562175.19fccbe3064ee0df9e08299c115dd3c2.', STARTKEY => '', ENDKEY => ''} 2024-12-09T13:22:43,387 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733750562175.19fccbe3064ee0df9e08299c115dd3c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:43,387 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 19fccbe3064ee0df9e08299c115dd3c2 2024-12-09T13:22:43,387 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 19fccbe3064ee0df9e08299c115dd3c2 2024-12-09T13:22:43,394 INFO [StoreOpener-19fccbe3064ee0df9e08299c115dd3c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 19fccbe3064ee0df9e08299c115dd3c2 2024-12-09T13:22:43,395 INFO [StoreOpener-19fccbe3064ee0df9e08299c115dd3c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 19fccbe3064ee0df9e08299c115dd3c2 columnFamilyName a 2024-12-09T13:22:43,395 DEBUG [StoreOpener-19fccbe3064ee0df9e08299c115dd3c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:43,396 INFO [StoreOpener-19fccbe3064ee0df9e08299c115dd3c2-1 {}] regionserver.HStore(327): Store=19fccbe3064ee0df9e08299c115dd3c2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:43,396 INFO [StoreOpener-19fccbe3064ee0df9e08299c115dd3c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 19fccbe3064ee0df9e08299c115dd3c2 2024-12-09T13:22:43,398 INFO [StoreOpener-19fccbe3064ee0df9e08299c115dd3c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 19fccbe3064ee0df9e08299c115dd3c2 columnFamilyName b 2024-12-09T13:22:43,398 DEBUG [StoreOpener-19fccbe3064ee0df9e08299c115dd3c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:43,399 INFO [StoreOpener-19fccbe3064ee0df9e08299c115dd3c2-1 {}] regionserver.HStore(327): Store=19fccbe3064ee0df9e08299c115dd3c2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:43,399 INFO [StoreOpener-19fccbe3064ee0df9e08299c115dd3c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 19fccbe3064ee0df9e08299c115dd3c2 2024-12-09T13:22:43,400 INFO [StoreOpener-19fccbe3064ee0df9e08299c115dd3c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 19fccbe3064ee0df9e08299c115dd3c2 columnFamilyName c 2024-12-09T13:22:43,400 DEBUG [StoreOpener-19fccbe3064ee0df9e08299c115dd3c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:43,403 INFO [StoreOpener-19fccbe3064ee0df9e08299c115dd3c2-1 {}] regionserver.HStore(327): Store=19fccbe3064ee0df9e08299c115dd3c2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:43,404 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 19fccbe3064ee0df9e08299c115dd3c2 2024-12-09T13:22:43,409 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2 2024-12-09T13:22:43,412 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2 2024-12-09T13:22:43,414 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/0000000000000003000 2024-12-09T13:22:43,456 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/0000000000000003000 2024-12-09T13:22:43,457 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/0000000000000006000 2024-12-09T13:22:43,501 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/0000000000000006000 2024-12-09T13:22:43,502 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 19fccbe3064ee0df9e08299c115dd3c2 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-09T13:22:43,562 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/.tmp/a/c1e4f71abb54463abfde9c4c532dcb22 is 41, key is test2727/a:100/1733750562919/Put/seqid=0 2024-12-09T13:22:43,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741871_1048 (size=84227) 2024-12-09T13:22:43,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741871_1048 (size=84227) 2024-12-09T13:22:43,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741871_1048 (size=84227) 2024-12-09T13:22:43,607 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/.tmp/a/c1e4f71abb54463abfde9c4c532dcb22 2024-12-09T13:22:43,656 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/.tmp/b/0776d40e09574586a1ab9b1b3ff66879 is 41, key is test2727/b:100/1733750562988/Put/seqid=0 2024-12-09T13:22:43,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741872_1049 (size=84609) 2024-12-09T13:22:43,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741872_1049 (size=84609) 2024-12-09T13:22:43,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741872_1049 (size=84609) 2024-12-09T13:22:43,714 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/.tmp/b/0776d40e09574586a1ab9b1b3ff66879 2024-12-09T13:22:43,748 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/.tmp/c/4fb4d6aaf5584ff9a136aa68c204f8af is 41, key is test2727/c:100/1733750563056/Put/seqid=0 2024-12-09T13:22:43,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741873_1050 (size=84609) 2024-12-09T13:22:43,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741873_1050 (size=84609) 2024-12-09T13:22:43,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741873_1050 (size=84609) 2024-12-09T13:22:43,791 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/.tmp/c/4fb4d6aaf5584ff9a136aa68c204f8af 2024-12-09T13:22:43,814 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/.tmp/a/c1e4f71abb54463abfde9c4c532dcb22 as hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/a/c1e4f71abb54463abfde9c4c532dcb22 2024-12-09T13:22:43,828 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/a/c1e4f71abb54463abfde9c4c532dcb22, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-09T13:22:43,830 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/.tmp/b/0776d40e09574586a1ab9b1b3ff66879 as hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/b/0776d40e09574586a1ab9b1b3ff66879 2024-12-09T13:22:43,842 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/b/0776d40e09574586a1ab9b1b3ff66879, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-09T13:22:43,843 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/.tmp/c/4fb4d6aaf5584ff9a136aa68c204f8af as hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/c/4fb4d6aaf5584ff9a136aa68c204f8af 2024-12-09T13:22:43,853 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/c/4fb4d6aaf5584ff9a136aa68c204f8af, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-09T13:22:43,853 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 19fccbe3064ee0df9e08299c115dd3c2 in 352ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-09T13:22:43,854 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/0000000000000003000 2024-12-09T13:22:43,855 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/0000000000000006000 2024-12-09T13:22:43,856 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 19fccbe3064ee0df9e08299c115dd3c2 2024-12-09T13:22:43,856 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 19fccbe3064ee0df9e08299c115dd3c2 2024-12-09T13:22:43,857 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T13:22:43,860 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 19fccbe3064ee0df9e08299c115dd3c2 2024-12-09T13:22:43,867 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/hbase/data/default/test2727/19fccbe3064ee0df9e08299c115dd3c2/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-09T13:22:43,870 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 19fccbe3064ee0df9e08299c115dd3c2; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60769200, jitterRate=-0.09446835517883301}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T13:22:43,871 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 19fccbe3064ee0df9e08299c115dd3c2: Writing region info on filesystem at 1733750563387Initializing all the Stores at 1733750563393 (+6 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750563393Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750563394 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750563394Obtaining lock to block concurrent updates at 1733750563502 (+108 ms)Preparing flush snapshotting stores in 19fccbe3064ee0df9e08299c115dd3c2 at 1733750563502Finished memstore snapshotting test2727,,1733750562175.19fccbe3064ee0df9e08299c115dd3c2., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733750563502Flushing stores of test2727,,1733750562175.19fccbe3064ee0df9e08299c115dd3c2. at 1733750563502Flushing 19fccbe3064ee0df9e08299c115dd3c2/a: creating writer at 1733750563502Flushing 19fccbe3064ee0df9e08299c115dd3c2/a: appending metadata at 1733750563558 (+56 ms)Flushing 19fccbe3064ee0df9e08299c115dd3c2/a: closing flushed file at 1733750563558Flushing 19fccbe3064ee0df9e08299c115dd3c2/b: creating writer at 1733750563616 (+58 ms)Flushing 19fccbe3064ee0df9e08299c115dd3c2/b: appending metadata at 1733750563654 (+38 ms)Flushing 19fccbe3064ee0df9e08299c115dd3c2/b: closing flushed file at 1733750563654Flushing 19fccbe3064ee0df9e08299c115dd3c2/c: creating writer at 1733750563721 (+67 ms)Flushing 19fccbe3064ee0df9e08299c115dd3c2/c: appending metadata at 1733750563747 (+26 ms)Flushing 19fccbe3064ee0df9e08299c115dd3c2/c: closing flushed file at 1733750563747Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1344261b: reopening flushed file at 1733750563812 (+65 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@cb1abc4: reopening flushed file at 1733750563828 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@472a0ccc: reopening flushed file at 1733750563842 (+14 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 19fccbe3064ee0df9e08299c115dd3c2 in 352ms, sequenceid=6000, compaction requested=false; wal=null at 1733750563853 (+11 ms)Cleaning up temporary data from old regions at 1733750563856 (+3 ms)Region opened successfully at 1733750563871 (+15 ms) 2024-12-09T13:22:43,873 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-09T13:22:43,873 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 19fccbe3064ee0df9e08299c115dd3c2, disabling compactions & flushes 2024-12-09T13:22:43,873 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733750562175.19fccbe3064ee0df9e08299c115dd3c2. 2024-12-09T13:22:43,873 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733750562175.19fccbe3064ee0df9e08299c115dd3c2. 2024-12-09T13:22:43,873 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733750562175.19fccbe3064ee0df9e08299c115dd3c2. after waiting 0 ms 2024-12-09T13:22:43,873 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733750562175.19fccbe3064ee0df9e08299c115dd3c2. 2024-12-09T13:22:43,882 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733750562175.19fccbe3064ee0df9e08299c115dd3c2. 2024-12-09T13:22:43,883 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 19fccbe3064ee0df9e08299c115dd3c2: Waiting for close lock at 1733750563873Disabling compacts and flushes for region at 1733750563873Disabling writes for close at 1733750563873Writing region close event to WAL at 1733750563882 (+9 ms)Closed at 1733750563882 2024-12-09T13:22:43,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741870_1047 (size=93) 2024-12-09T13:22:43,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741870_1047 (size=93) 2024-12-09T13:22:43,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741870_1047 (size=93) 2024-12-09T13:22:43,898 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T13:22:43,898 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733750563353) 2024-12-09T13:22:43,914 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#test2727 Thread=396 (was 394) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/cluster_b6088222-ea39-2e85-344a-dadc86871089/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:49414 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:53362 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/cluster_b6088222-ea39-2e85-344a-dadc86871089/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:54068 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=921 (was 861) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=668 (was 683), ProcessCount=11 (was 11), AvailableMemoryMB=3713 (was 3837) 2024-12-09T13:22:43,932 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=396, OpenFileDescriptor=921, MaxFileDescriptor=1048576, SystemLoadAverage=668, ProcessCount=11, AvailableMemoryMB=3713 2024-12-09T13:22:43,956 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:22:43,964 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733750563955, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:22:43,965 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733750563964 2024-12-09T13:22:43,977 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733750563955/wal.1733750563964 2024-12-09T13:22:43,983 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34135:34135),(127.0.0.1/127.0.0.1:34323:34323),(127.0.0.1/127.0.0.1:44161:44161)] 2024-12-09T13:22:43,987 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2aa49b631d634cc7ee216f59d957947e, NAME => 'testSequentialEditLogSeqNum,,1733750563956.2aa49b631d634cc7ee216f59d957947e.', STARTKEY => '', ENDKEY => ''} 2024-12-09T13:22:43,987 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733750563956.2aa49b631d634cc7ee216f59d957947e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:43,987 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2aa49b631d634cc7ee216f59d957947e 2024-12-09T13:22:43,988 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2aa49b631d634cc7ee216f59d957947e 2024-12-09T13:22:43,989 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:41625/hbase/data/default/testSequentialEditLogSeqNum/2aa49b631d634cc7ee216f59d957947e doesn't exist for region: 2aa49b631d634cc7ee216f59d957947e on table testSequentialEditLogSeqNum 2024-12-09T13:22:43,990 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 2aa49b631d634cc7ee216f59d957947e on table testSequentialEditLogSeqNum 2024-12-09T13:22:44,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741875_1052 (size=62) 2024-12-09T13:22:44,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741875_1052 (size=62) 2024-12-09T13:22:44,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741875_1052 (size=62) 2024-12-09T13:22:44,022 INFO [StoreOpener-2aa49b631d634cc7ee216f59d957947e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2aa49b631d634cc7ee216f59d957947e 2024-12-09T13:22:44,024 INFO [StoreOpener-2aa49b631d634cc7ee216f59d957947e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2aa49b631d634cc7ee216f59d957947e columnFamilyName a 2024-12-09T13:22:44,025 DEBUG [StoreOpener-2aa49b631d634cc7ee216f59d957947e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:44,025 INFO [StoreOpener-2aa49b631d634cc7ee216f59d957947e-1 {}] regionserver.HStore(327): Store=2aa49b631d634cc7ee216f59d957947e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:44,025 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2aa49b631d634cc7ee216f59d957947e 2024-12-09T13:22:44,026 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testSequentialEditLogSeqNum/2aa49b631d634cc7ee216f59d957947e 2024-12-09T13:22:44,026 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testSequentialEditLogSeqNum/2aa49b631d634cc7ee216f59d957947e 2024-12-09T13:22:44,027 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2aa49b631d634cc7ee216f59d957947e 2024-12-09T13:22:44,027 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2aa49b631d634cc7ee216f59d957947e 2024-12-09T13:22:44,029 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2aa49b631d634cc7ee216f59d957947e 2024-12-09T13:22:44,033 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/hbase/data/default/testSequentialEditLogSeqNum/2aa49b631d634cc7ee216f59d957947e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T13:22:44,033 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2aa49b631d634cc7ee216f59d957947e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59272680, jitterRate=-0.1167682409286499}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T13:22:44,035 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2aa49b631d634cc7ee216f59d957947e: Writing region info on filesystem at 1733750563988Initializing all the Stores at 1733750564020 (+32 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750564020Cleaning up temporary data from old regions at 1733750564027 (+7 ms)Region opened successfully at 1733750564034 (+7 ms) 2024-12-09T13:22:44,049 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2aa49b631d634cc7ee216f59d957947e 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-09T13:22:44,083 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testSequentialEditLogSeqNum/2aa49b631d634cc7ee216f59d957947e/.tmp/a/1e996bb768244096b5e6d20ba596d484 is 81, key is testSequentialEditLogSeqNum/a:x0/1733750564035/Put/seqid=0 2024-12-09T13:22:44,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741876_1053 (size=5833) 2024-12-09T13:22:44,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741876_1053 (size=5833) 2024-12-09T13:22:44,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741876_1053 (size=5833) 2024-12-09T13:22:44,182 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/testSequentialEditLogSeqNum/2aa49b631d634cc7ee216f59d957947e/.tmp/a/1e996bb768244096b5e6d20ba596d484 2024-12-09T13:22:44,219 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testSequentialEditLogSeqNum/2aa49b631d634cc7ee216f59d957947e/.tmp/a/1e996bb768244096b5e6d20ba596d484 as hdfs://localhost:41625/hbase/data/default/testSequentialEditLogSeqNum/2aa49b631d634cc7ee216f59d957947e/a/1e996bb768244096b5e6d20ba596d484 2024-12-09T13:22:44,250 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/testSequentialEditLogSeqNum/2aa49b631d634cc7ee216f59d957947e/a/1e996bb768244096b5e6d20ba596d484, entries=10, sequenceid=13, filesize=5.7 K 2024-12-09T13:22:44,252 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 2aa49b631d634cc7ee216f59d957947e in 204ms, sequenceid=13, compaction requested=false 2024-12-09T13:22:44,252 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2aa49b631d634cc7ee216f59d957947e: 2024-12-09T13:22:44,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T13:22:44,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T13:22:44,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T13:22:44,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T13:22:44,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T13:22:44,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741874_1051 (size=3017) 2024-12-09T13:22:44,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741874_1051 (size=3017) 2024-12-09T13:22:44,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741874_1051 (size=3017) 2024-12-09T13:22:44,307 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41625/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733750563955/wal.1733750563964, size=2.9 K (3017bytes) 2024-12-09T13:22:44,307 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41625/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733750563955/wal.1733750563964 2024-12-09T13:22:44,307 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41625/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733750563955/wal.1733750563964 after 0ms 2024-12-09T13:22:44,311 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41625/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733750563955/wal.1733750563964 took 5ms 2024-12-09T13:22:44,314 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41625/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733750563955/wal.1733750563964 so closing down 2024-12-09T13:22:44,314 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T13:22:44,323 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733750563964.temp 2024-12-09T13:22:44,328 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testSequentialEditLogSeqNum/2aa49b631d634cc7ee216f59d957947e/recovered.edits/0000000000000000003-wal.1733750563964.temp 2024-12-09T13:22:44,330 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T13:22:44,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741877_1054 (size=2433) 2024-12-09T13:22:44,354 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testSequentialEditLogSeqNum/2aa49b631d634cc7ee216f59d957947e/recovered.edits/0000000000000000003-wal.1733750563964.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-09T13:22:44,356 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41625/hbase/data/default/testSequentialEditLogSeqNum/2aa49b631d634cc7ee216f59d957947e/recovered.edits/0000000000000000003-wal.1733750563964.temp to hdfs://localhost:41625/hbase/data/default/testSequentialEditLogSeqNum/2aa49b631d634cc7ee216f59d957947e/recovered.edits/0000000000000000020 2024-12-09T13:22:44,357 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 45 ms; skipped=2; WAL=hdfs://localhost:41625/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733750563955/wal.1733750563964, size=2.9 K, length=3017, corrupted=false, cancelled=false 2024-12-09T13:22:44,357 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41625/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733750563955/wal.1733750563964, journal: Splitting hdfs://localhost:41625/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733750563955/wal.1733750563964, size=2.9 K (3017bytes) at 1733750564307Finishing writing output for hdfs://localhost:41625/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733750563955/wal.1733750563964 so closing down at 1733750564314 (+7 ms)Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testSequentialEditLogSeqNum/2aa49b631d634cc7ee216f59d957947e/recovered.edits/0000000000000000003-wal.1733750563964.temp at 1733750564328 (+14 ms)3 split writer threads finished at 1733750564330 (+2 ms)Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testSequentialEditLogSeqNum/2aa49b631d634cc7ee216f59d957947e/recovered.edits/0000000000000000003-wal.1733750563964.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733750564354 (+24 ms)Rename recovered edits hdfs://localhost:41625/hbase/data/default/testSequentialEditLogSeqNum/2aa49b631d634cc7ee216f59d957947e/recovered.edits/0000000000000000003-wal.1733750563964.temp to hdfs://localhost:41625/hbase/data/default/testSequentialEditLogSeqNum/2aa49b631d634cc7ee216f59d957947e/recovered.edits/0000000000000000020 at 1733750564356 (+2 ms)Processed 17 edits across 1 Regions in 45 ms; skipped=2; WAL=hdfs://localhost:41625/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733750563955/wal.1733750563964, size=2.9 K, length=3017, corrupted=false, cancelled=false at 1733750564357 (+1 ms) 2024-12-09T13:22:44,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741877_1054 (size=2433) 2024-12-09T13:22:44,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741877_1054 (size=2433) 2024-12-09T13:22:44,382 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=401 (was 396) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:53362 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=957 (was 921) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=668 (was 668), ProcessCount=11 (was 11), AvailableMemoryMB=3697 (was 3713) 2024-12-09T13:22:44,398 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=401, OpenFileDescriptor=957, MaxFileDescriptor=1048576, SystemLoadAverage=668, ProcessCount=11, AvailableMemoryMB=3696 2024-12-09T13:22:44,415 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:22:44,427 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:22:44,451 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T13:22:44,454 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-60651642, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/hregion-60651642, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:22:44,475 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-60651642/hregion-60651642.1733750564455, exclude list is [], retry=0 2024-12-09T13:22:44,480 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:22:44,483 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:22:44,484 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:22:44,491 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-60651642/hregion-60651642.1733750564455 2024-12-09T13:22:44,494 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34135:34135),(127.0.0.1/127.0.0.1:34323:34323)] 2024-12-09T13:22:44,495 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 693b4565571d28e326ca5c8812ff35c8, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733750564418.693b4565571d28e326ca5c8812ff35c8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41625/hbase 2024-12-09T13:22:44,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741879_1056 (size=70) 2024-12-09T13:22:44,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741879_1056 (size=70) 2024-12-09T13:22:44,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741879_1056 (size=70) 2024-12-09T13:22:44,523 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733750564418.693b4565571d28e326ca5c8812ff35c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:44,527 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:44,530 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 693b4565571d28e326ca5c8812ff35c8 columnFamilyName a 2024-12-09T13:22:44,530 DEBUG [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:44,532 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] regionserver.HStore(327): Store=693b4565571d28e326ca5c8812ff35c8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:44,532 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:44,534 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 693b4565571d28e326ca5c8812ff35c8 columnFamilyName b 2024-12-09T13:22:44,534 DEBUG [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:44,535 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] regionserver.HStore(327): Store=693b4565571d28e326ca5c8812ff35c8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:44,535 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:44,537 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 693b4565571d28e326ca5c8812ff35c8 columnFamilyName c 2024-12-09T13:22:44,537 DEBUG [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:44,538 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] regionserver.HStore(327): Store=693b4565571d28e326ca5c8812ff35c8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:44,538 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:44,539 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:44,540 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:44,541 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:44,541 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:44,544 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T13:22:44,548 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:44,550 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T13:22:44,551 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 693b4565571d28e326ca5c8812ff35c8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61045865, jitterRate=-0.09034572541713715}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T13:22:44,552 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 693b4565571d28e326ca5c8812ff35c8: Writing region info on filesystem at 1733750564523Initializing all the Stores at 1733750564524 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750564524Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750564527 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750564527Cleaning up temporary data from old regions at 1733750564541 (+14 ms)Region opened successfully at 1733750564552 (+11 ms) 2024-12-09T13:22:44,553 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 693b4565571d28e326ca5c8812ff35c8, disabling compactions & flushes 2024-12-09T13:22:44,553 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733750564418.693b4565571d28e326ca5c8812ff35c8. 2024-12-09T13:22:44,553 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733750564418.693b4565571d28e326ca5c8812ff35c8. 2024-12-09T13:22:44,553 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733750564418.693b4565571d28e326ca5c8812ff35c8. after waiting 0 ms 2024-12-09T13:22:44,553 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733750564418.693b4565571d28e326ca5c8812ff35c8. 2024-12-09T13:22:44,553 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733750564418.693b4565571d28e326ca5c8812ff35c8. 2024-12-09T13:22:44,554 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 693b4565571d28e326ca5c8812ff35c8: Waiting for close lock at 1733750564552Disabling compacts and flushes for region at 1733750564552Disabling writes for close at 1733750564553 (+1 ms)Writing region close event to WAL at 1733750564553Closed at 1733750564553 2024-12-09T13:22:44,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741878_1055 (size=93) 2024-12-09T13:22:44,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741878_1055 (size=93) 2024-12-09T13:22:44,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741878_1055 (size=93) 2024-12-09T13:22:44,579 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T13:22:44,579 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-60651642:(num 1733750564455) 2024-12-09T13:22:44,579 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T13:22:44,581 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733750564414, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:22:44,599 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733750564414/wal.1733750564582, exclude list is [], retry=0 2024-12-09T13:22:44,604 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:22:44,605 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:22:44,605 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:22:44,614 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733750564414/wal.1733750564582 2024-12-09T13:22:44,615 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34323:34323),(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34135:34135)] 2024-12-09T13:22:44,615 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 693b4565571d28e326ca5c8812ff35c8, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733750564418.693b4565571d28e326ca5c8812ff35c8.', STARTKEY => '', ENDKEY => ''} 2024-12-09T13:22:44,615 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733750564418.693b4565571d28e326ca5c8812ff35c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:44,615 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:44,615 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:44,621 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:44,623 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 693b4565571d28e326ca5c8812ff35c8 columnFamilyName a 2024-12-09T13:22:44,623 DEBUG [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:44,626 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] regionserver.HStore(327): Store=693b4565571d28e326ca5c8812ff35c8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:44,627 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:44,628 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 693b4565571d28e326ca5c8812ff35c8 columnFamilyName b 2024-12-09T13:22:44,628 DEBUG [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:44,628 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] regionserver.HStore(327): Store=693b4565571d28e326ca5c8812ff35c8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:44,628 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:44,629 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 693b4565571d28e326ca5c8812ff35c8 columnFamilyName c 2024-12-09T13:22:44,629 DEBUG [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:44,630 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] regionserver.HStore(327): Store=693b4565571d28e326ca5c8812ff35c8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:44,630 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:44,631 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:44,632 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:44,634 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:44,634 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:44,635 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T13:22:44,653 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:44,660 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 693b4565571d28e326ca5c8812ff35c8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73061303, jitterRate=0.08869825303554535}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T13:22:44,662 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 693b4565571d28e326ca5c8812ff35c8: Writing region info on filesystem at 1733750564616Initializing all the Stores at 1733750564617 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750564617Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750564621 (+4 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750564621Cleaning up temporary data from old regions at 1733750564634 (+13 ms)Region opened successfully at 1733750564662 (+28 ms) 2024-12-09T13:22:44,668 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733750564667/Put/seqid=0 2024-12-09T13:22:44,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741881_1058 (size=4826) 2024-12-09T13:22:44,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741881_1058 (size=4826) 2024-12-09T13:22:44,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741881_1058 (size=4826) 2024-12-09T13:22:44,691 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41625/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 693b4565571d28e326ca5c8812ff35c8/a 2024-12-09T13:22:44,702 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-09T13:22:44,702 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T13:22:44,702 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 693b4565571d28e326ca5c8812ff35c8: 2024-12-09T13:22:44,709 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8/a/3f935bb2b3bb4325acfaa2ce1b149f57_SeqId_3_ 2024-12-09T13:22:44,710 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41625/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 693b4565571d28e326ca5c8812ff35c8/a as hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8/a/3f935bb2b3bb4325acfaa2ce1b149f57_SeqId_3_ - updating store file list. 2024-12-09T13:22:44,716 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3f935bb2b3bb4325acfaa2ce1b149f57_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-09T13:22:44,717 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8/a/3f935bb2b3bb4325acfaa2ce1b149f57_SeqId_3_ into 693b4565571d28e326ca5c8812ff35c8/a 2024-12-09T13:22:44,717 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41625/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 693b4565571d28e326ca5c8812ff35c8/a (new location: hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8/a/3f935bb2b3bb4325acfaa2ce1b149f57_SeqId_3_) 2024-12-09T13:22:44,774 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41625/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733750564414/wal.1733750564582, size=0 (0bytes) 2024-12-09T13:22:44,774 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41625/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733750564414/wal.1733750564582 might be still open, length is 0 2024-12-09T13:22:44,774 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41625/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733750564414/wal.1733750564582 2024-12-09T13:22:44,775 WARN [IPC Server handler 4 on default port 41625 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733750564414/wal.1733750564582 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-09T13:22:44,775 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41625/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733750564414/wal.1733750564582 after 1ms 2024-12-09T13:22:46,275 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:53468 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:40963:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53468 dst: /127.0.0.1:40963 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:40963 remote=/127.0.0.1:53468]. Total timeout mills is 60000, 58452 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:22:46,277 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:49496 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37811:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49496 dst: /127.0.0.1:37811 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:22:46,277 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:54262 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:41523:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54262 dst: /127.0.0.1:41523 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:22:46,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741880_1059 (size=504) 2024-12-09T13:22:46,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741880_1059 (size=504) 2024-12-09T13:22:46,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741880_1059 (size=504) 2024-12-09T13:22:47,499 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-09T13:22:47,499 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-09T13:22:47,500 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-09T13:22:47,501 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-09T13:22:48,776 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41625/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733750564414/wal.1733750564582 after 4002ms 2024-12-09T13:22:48,782 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41625/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733750564414/wal.1733750564582 took 4008ms 2024-12-09T13:22:48,805 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41625/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733750564414/wal.1733750564582; continuing. 2024-12-09T13:22:48,805 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41625/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733750564414/wal.1733750564582 so closing down 2024-12-09T13:22:48,805 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T13:22:48,834 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733750564582.temp 2024-12-09T13:22:48,848 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8/recovered.edits/0000000000000000005-wal.1733750564582.temp 2024-12-09T13:22:48,850 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T13:22:48,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741882_1060 (size=238) 2024-12-09T13:22:48,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741882_1060 (size=238) 2024-12-09T13:22:48,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741882_1060 (size=238) 2024-12-09T13:22:48,890 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8/recovered.edits/0000000000000000005-wal.1733750564582.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-09T13:22:48,892 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8/recovered.edits/0000000000000000005-wal.1733750564582.temp to hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8/recovered.edits/0000000000000000005 2024-12-09T13:22:48,892 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 98 ms; skipped=1; WAL=hdfs://localhost:41625/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733750564414/wal.1733750564582, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T13:22:48,892 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41625/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733750564414/wal.1733750564582, journal: Splitting hdfs://localhost:41625/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733750564414/wal.1733750564582, size=0 (0bytes) at 1733750564774Finishing writing output for hdfs://localhost:41625/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733750564414/wal.1733750564582 so closing down at 1733750568805 (+4031 ms)Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8/recovered.edits/0000000000000000005-wal.1733750564582.temp at 1733750568848 (+43 ms)3 split writer threads finished at 1733750568850 (+2 ms)Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8/recovered.edits/0000000000000000005-wal.1733750564582.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733750568890 (+40 ms)Rename recovered edits hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8/recovered.edits/0000000000000000005-wal.1733750564582.temp to hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8/recovered.edits/0000000000000000005 at 1733750568892 (+2 ms)Processed 2 edits across 1 Regions in 98 ms; skipped=1; WAL=hdfs://localhost:41625/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733750564414/wal.1733750564582, size=0, length=0, corrupted=false, cancelled=false at 1733750568892 2024-12-09T13:22:48,894 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41625/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733750564414/wal.1733750564582 to hdfs://localhost:41625/hbase/oldWALs/wal.1733750564582 2024-12-09T13:22:48,895 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8/recovered.edits/0000000000000000005 2024-12-09T13:22:48,896 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T13:22:48,898 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733750564414, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:22:48,917 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733750564414/wal.1733750568898, exclude list is [], retry=0 2024-12-09T13:22:48,920 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:22:48,921 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:22:48,921 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:22:48,940 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733750564414/wal.1733750568898 2024-12-09T13:22:48,942 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34323:34323),(127.0.0.1/127.0.0.1:34135:34135)] 2024-12-09T13:22:48,942 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 693b4565571d28e326ca5c8812ff35c8, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733750564418.693b4565571d28e326ca5c8812ff35c8.', STARTKEY => '', ENDKEY => ''} 2024-12-09T13:22:48,942 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733750564418.693b4565571d28e326ca5c8812ff35c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:48,942 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:48,942 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:48,945 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:48,947 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 693b4565571d28e326ca5c8812ff35c8 columnFamilyName a 2024-12-09T13:22:48,947 DEBUG [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:48,956 DEBUG [StoreFileOpener-693b4565571d28e326ca5c8812ff35c8-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3f935bb2b3bb4325acfaa2ce1b149f57_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-09T13:22:48,956 DEBUG [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8/a/3f935bb2b3bb4325acfaa2ce1b149f57_SeqId_3_ 2024-12-09T13:22:48,956 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] regionserver.HStore(327): Store=693b4565571d28e326ca5c8812ff35c8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:48,957 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:48,958 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 693b4565571d28e326ca5c8812ff35c8 columnFamilyName b 2024-12-09T13:22:48,958 DEBUG [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:48,958 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] regionserver.HStore(327): Store=693b4565571d28e326ca5c8812ff35c8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:48,958 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:48,960 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 693b4565571d28e326ca5c8812ff35c8 columnFamilyName c 2024-12-09T13:22:48,960 DEBUG [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:48,960 INFO [StoreOpener-693b4565571d28e326ca5c8812ff35c8-1 {}] regionserver.HStore(327): Store=693b4565571d28e326ca5c8812ff35c8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:48,960 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:48,961 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:48,963 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:48,964 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8/recovered.edits/0000000000000000005 2024-12-09T13:22:48,968 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8/recovered.edits/0000000000000000005 2024-12-09T13:22:48,968 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 693b4565571d28e326ca5c8812ff35c8 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-09T13:22:48,987 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8/.tmp/a/dc80a59e0d2f4eeb84cf6cdbf8d7eec1 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733750564722/Put/seqid=0 2024-12-09T13:22:49,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741884_1062 (size=5149) 2024-12-09T13:22:49,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741884_1062 (size=5149) 2024-12-09T13:22:49,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741884_1062 (size=5149) 2024-12-09T13:22:49,021 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8/.tmp/a/dc80a59e0d2f4eeb84cf6cdbf8d7eec1 2024-12-09T13:22:49,035 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8/.tmp/a/dc80a59e0d2f4eeb84cf6cdbf8d7eec1 as hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8/a/dc80a59e0d2f4eeb84cf6cdbf8d7eec1 2024-12-09T13:22:49,043 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8/a/dc80a59e0d2f4eeb84cf6cdbf8d7eec1, entries=1, sequenceid=5, filesize=5.0 K 2024-12-09T13:22:49,044 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 693b4565571d28e326ca5c8812ff35c8 in 75ms, sequenceid=5, compaction requested=false; wal=null 2024-12-09T13:22:49,044 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8/recovered.edits/0000000000000000005 2024-12-09T13:22:49,050 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:49,050 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:49,058 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T13:22:49,073 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 693b4565571d28e326ca5c8812ff35c8 2024-12-09T13:22:49,078 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/693b4565571d28e326ca5c8812ff35c8/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-09T13:22:49,080 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 693b4565571d28e326ca5c8812ff35c8; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64900079, jitterRate=-0.032913461327552795}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T13:22:49,080 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 693b4565571d28e326ca5c8812ff35c8: Writing region info on filesystem at 1733750568943Initializing all the Stores at 1733750568944 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750568944Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750568944Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750568944Obtaining lock to block concurrent updates at 1733750568968 (+24 ms)Preparing flush snapshotting stores in 693b4565571d28e326ca5c8812ff35c8 at 1733750568968Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733750564418.693b4565571d28e326ca5c8812ff35c8., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733750568968Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733750564418.693b4565571d28e326ca5c8812ff35c8. at 1733750568968Flushing 693b4565571d28e326ca5c8812ff35c8/a: creating writer at 1733750568968Flushing 693b4565571d28e326ca5c8812ff35c8/a: appending metadata at 1733750568986 (+18 ms)Flushing 693b4565571d28e326ca5c8812ff35c8/a: closing flushed file at 1733750568986Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@514f8867: reopening flushed file at 1733750569031 (+45 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 693b4565571d28e326ca5c8812ff35c8 in 75ms, sequenceid=5, compaction requested=false; wal=null at 1733750569044 (+13 ms)Cleaning up temporary data from old regions at 1733750569050 (+6 ms)Region opened successfully at 1733750569080 (+30 ms) 2024-12-09T13:22:49,086 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 693b4565571d28e326ca5c8812ff35c8, disabling compactions & flushes 2024-12-09T13:22:49,086 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733750564418.693b4565571d28e326ca5c8812ff35c8. 2024-12-09T13:22:49,086 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733750564418.693b4565571d28e326ca5c8812ff35c8. 2024-12-09T13:22:49,086 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733750564418.693b4565571d28e326ca5c8812ff35c8. after waiting 0 ms 2024-12-09T13:22:49,086 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733750564418.693b4565571d28e326ca5c8812ff35c8. 2024-12-09T13:22:49,098 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733750564418.693b4565571d28e326ca5c8812ff35c8. 2024-12-09T13:22:49,098 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 693b4565571d28e326ca5c8812ff35c8: Waiting for close lock at 1733750569086Disabling compacts and flushes for region at 1733750569086Disabling writes for close at 1733750569086Writing region close event to WAL at 1733750569098 (+12 ms)Closed at 1733750569098 2024-12-09T13:22:49,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741883_1061 (size=93) 2024-12-09T13:22:49,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741883_1061 (size=93) 2024-12-09T13:22:49,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741883_1061 (size=93) 2024-12-09T13:22:49,115 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T13:22:49,115 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733750568898) 2024-12-09T13:22:49,137 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=408 (was 401) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1263169559_22 at /127.0.0.1:35588 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (587680655) connection to localhost/127.0.0.1:41625 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1263169559_22 at /127.0.0.1:43636 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1263169559_22 at /127.0.0.1:41352 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:41625 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 957) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=727 (was 668) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3659 (was 3696) 2024-12-09T13:22:49,157 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=408, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=727, ProcessCount=11, AvailableMemoryMB=3658 2024-12-09T13:22:49,176 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:22:49,182 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T13:22:49,188 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is cad571b91d50,44823,1733750546819 2024-12-09T13:22:49,191 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@66b1cac7 2024-12-09T13:22:49,192 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T13:22:49,194 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T13:22:49,198 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44823 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T13:22:49,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44823 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-09T13:22:49,215 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T13:22:49,218 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:49,219 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44823 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-09T13:22:49,221 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T13:22:49,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T13:22:49,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741885_1063 (size=694) 2024-12-09T13:22:49,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741885_1063 (size=694) 2024-12-09T13:22:49,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741885_1063 (size=694) 2024-12-09T13:22:49,248 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f4be257e1d5e695afc605404060499d5, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1 2024-12-09T13:22:49,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741886_1064 (size=77) 2024-12-09T13:22:49,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741886_1064 (size=77) 2024-12-09T13:22:49,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741886_1064 (size=77) 2024-12-09T13:22:49,265 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:49,266 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing f4be257e1d5e695afc605404060499d5, disabling compactions & flushes 2024-12-09T13:22:49,266 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:49,266 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:49,266 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. after waiting 0 ms 2024-12-09T13:22:49,266 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:49,266 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:49,266 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for f4be257e1d5e695afc605404060499d5: Waiting for close lock at 1733750569266Disabling compacts and flushes for region at 1733750569266Disabling writes for close at 1733750569266Writing region close event to WAL at 1733750569266Closed at 1733750569266 2024-12-09T13:22:49,268 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T13:22:49,274 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733750569268"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733750569268"}]},"ts":"1733750569268"} 2024-12-09T13:22:49,281 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T13:22:49,282 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T13:22:49,285 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733750569282"}]},"ts":"1733750569282"} 2024-12-09T13:22:49,290 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-09T13:22:49,290 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {cad571b91d50=0} racks are {/default-rack=0} 2024-12-09T13:22:49,292 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T13:22:49,292 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T13:22:49,292 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-09T13:22:49,292 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T13:22:49,292 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T13:22:49,292 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-09T13:22:49,292 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T13:22:49,292 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T13:22:49,292 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-09T13:22:49,292 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T13:22:49,293 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f4be257e1d5e695afc605404060499d5, ASSIGN}] 2024-12-09T13:22:49,298 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f4be257e1d5e695afc605404060499d5, ASSIGN 2024-12-09T13:22:49,299 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f4be257e1d5e695afc605404060499d5, ASSIGN; state=OFFLINE, location=cad571b91d50,34387,1733750548376; forceNewPlan=false, retain=false 2024-12-09T13:22:49,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T13:22:49,452 INFO [cad571b91d50:44823 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T13:22:49,453 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f4be257e1d5e695afc605404060499d5, regionState=OPENING, regionLocation=cad571b91d50,34387,1733750548376 2024-12-09T13:22:49,456 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f4be257e1d5e695afc605404060499d5, ASSIGN because future has completed 2024-12-09T13:22:49,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f4be257e1d5e695afc605404060499d5, server=cad571b91d50,34387,1733750548376}] 2024-12-09T13:22:49,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T13:22:49,611 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T13:22:49,613 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53161, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T13:22:49,623 INFO [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:49,623 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f4be257e1d5e695afc605404060499d5, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5.', STARTKEY => '', ENDKEY => ''} 2024-12-09T13:22:49,623 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:49,623 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:49,623 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:49,623 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:49,625 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:49,627 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4be257e1d5e695afc605404060499d5 columnFamilyName cf1 2024-12-09T13:22:49,627 DEBUG [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:49,630 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.HStore(327): Store=f4be257e1d5e695afc605404060499d5/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:49,630 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:49,632 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4be257e1d5e695afc605404060499d5 columnFamilyName cf2 2024-12-09T13:22:49,632 DEBUG [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:49,632 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.HStore(327): Store=f4be257e1d5e695afc605404060499d5/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:49,633 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:49,635 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:49,636 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:49,636 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:49,637 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:49,638 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T13:22:49,639 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:49,643 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T13:22:49,644 INFO [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f4be257e1d5e695afc605404060499d5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73156126, jitterRate=0.09011122584342957}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T13:22:49,644 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:49,644 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f4be257e1d5e695afc605404060499d5: Running coprocessor pre-open hook at 1733750569624Writing region info on filesystem at 1733750569624Initializing all the Stores at 1733750569625 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750569625Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750569625Cleaning up temporary data from old regions at 1733750569637 (+12 ms)Running coprocessor post-open hooks at 1733750569644 (+7 ms)Region opened successfully at 1733750569644 2024-12-09T13:22:49,647 INFO [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., pid=6, masterSystemTime=1733750569611 2024-12-09T13:22:49,654 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f4be257e1d5e695afc605404060499d5, regionState=OPEN, openSeqNum=2, regionLocation=cad571b91d50,34387,1733750548376 2024-12-09T13:22:49,654 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:49,656 INFO [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:49,658 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f4be257e1d5e695afc605404060499d5, server=cad571b91d50,34387,1733750548376 because future has completed 2024-12-09T13:22:49,659 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44823 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=cad571b91d50,34387,1733750548376, table=testReplayEditsAfterRegionMovedWithMultiCF, region=f4be257e1d5e695afc605404060499d5. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-09T13:22:49,665 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-09T13:22:49,665 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f4be257e1d5e695afc605404060499d5, server=cad571b91d50,34387,1733750548376 in 203 msec 2024-12-09T13:22:49,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-09T13:22:49,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f4be257e1d5e695afc605404060499d5, ASSIGN in 372 msec 2024-12-09T13:22:49,670 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T13:22:49,670 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733750569670"}]},"ts":"1733750569670"} 2024-12-09T13:22:49,674 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-09T13:22:49,675 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T13:22:49,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 475 msec 2024-12-09T13:22:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T13:22:49,860 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-09T13:22:49,860 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-09T13:22:49,862 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T13:22:49,867 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-09T13:22:49,868 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T13:22:49,868 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-09T13:22:49,880 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,34387,1733750548376, seqNum=2] 2024-12-09T13:22:49,882 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T13:22:49,884 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T13:22:49,901 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44823 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=f4be257e1d5e695afc605404060499d5, source=cad571b91d50,34387,1733750548376, destination=cad571b91d50,39675,1733750548162, warming up region on cad571b91d50,39675,1733750548162 2024-12-09T13:22:49,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44823 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T13:22:49,903 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44823 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=f4be257e1d5e695afc605404060499d5, source=cad571b91d50,34387,1733750548376, destination=cad571b91d50,39675,1733750548162, running balancer 2024-12-09T13:22:49,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42927, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T13:22:49,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44823 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f4be257e1d5e695afc605404060499d5, REOPEN/MOVE 2024-12-09T13:22:49,905 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f4be257e1d5e695afc605404060499d5, REOPEN/MOVE 2024-12-09T13:22:49,907 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f4be257e1d5e695afc605404060499d5, regionState=CLOSING, regionLocation=cad571b91d50,34387,1733750548376 2024-12-09T13:22:49,909 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39675 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:49,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39675 {}] regionserver.HRegion(7855): Warmup {ENCODED => f4be257e1d5e695afc605404060499d5, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5.', STARTKEY => '', ENDKEY => ''} 2024-12-09T13:22:49,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39675 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:49,909 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:49,910 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f4be257e1d5e695afc605404060499d5, REOPEN/MOVE because future has completed 2024-12-09T13:22:49,910 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T13:22:49,911 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure f4be257e1d5e695afc605404060499d5, server=cad571b91d50,34387,1733750548376}] 2024-12-09T13:22:49,911 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4be257e1d5e695afc605404060499d5 columnFamilyName cf1 2024-12-09T13:22:49,911 DEBUG [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:49,911 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.HStore(327): Store=f4be257e1d5e695afc605404060499d5/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:49,912 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:49,913 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4be257e1d5e695afc605404060499d5 columnFamilyName cf2 2024-12-09T13:22:49,913 DEBUG [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:49,913 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.HStore(327): Store=f4be257e1d5e695afc605404060499d5/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39675 {}] regionserver.HRegion(1722): Closing f4be257e1d5e695afc605404060499d5, disabling compactions & flushes 2024-12-09T13:22:49,914 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39675 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39675 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39675 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. after waiting 0 ms 2024-12-09T13:22:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39675 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:49,914 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39675 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39675 {}] regionserver.HRegion(1676): Region close journal for f4be257e1d5e695afc605404060499d5: Waiting for close lock at 1733750569914Disabling compacts and flushes for region at 1733750569914Disabling writes for close at 1733750569914Writing region close event to WAL at 1733750569914Closed at 1733750569914 2024-12-09T13:22:49,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44823 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-09T13:22:50,069 INFO [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:50,069 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T13:22:50,070 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing f4be257e1d5e695afc605404060499d5, disabling compactions & flushes 2024-12-09T13:22:50,070 INFO [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:50,070 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:50,070 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. after waiting 0 ms 2024-12-09T13:22:50,070 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:50,070 INFO [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing f4be257e1d5e695afc605404060499d5 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-09T13:22:50,087 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/.tmp/cf1/d1f05c7d445a43758e68ceb89e276978 is 35, key is r1/cf1:q/1733750569885/Put/seqid=0 2024-12-09T13:22:50,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741887_1065 (size=4783) 2024-12-09T13:22:50,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741887_1065 (size=4783) 2024-12-09T13:22:50,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741887_1065 (size=4783) 2024-12-09T13:22:50,096 INFO [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/.tmp/cf1/d1f05c7d445a43758e68ceb89e276978 2024-12-09T13:22:50,104 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/.tmp/cf1/d1f05c7d445a43758e68ceb89e276978 as hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf1/d1f05c7d445a43758e68ceb89e276978 2024-12-09T13:22:50,109 INFO [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf1/d1f05c7d445a43758e68ceb89e276978, entries=1, sequenceid=5, filesize=4.7 K 2024-12-09T13:22:50,111 INFO [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for f4be257e1d5e695afc605404060499d5 in 40ms, sequenceid=5, compaction requested=false 2024-12-09T13:22:50,111 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-09T13:22:50,116 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-09T13:22:50,119 INFO [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:50,119 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for f4be257e1d5e695afc605404060499d5: Waiting for close lock at 1733750570070Running coprocessor pre-close hooks at 1733750570070Disabling compacts and flushes for region at 1733750570070Disabling writes for close at 1733750570070Obtaining lock to block concurrent updates at 1733750570070Preparing flush snapshotting stores in f4be257e1d5e695afc605404060499d5 at 1733750570070Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733750570071 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. at 1733750570071Flushing f4be257e1d5e695afc605404060499d5/cf1: creating writer at 1733750570071Flushing f4be257e1d5e695afc605404060499d5/cf1: appending metadata at 1733750570087 (+16 ms)Flushing f4be257e1d5e695afc605404060499d5/cf1: closing flushed file at 1733750570087Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c5be22e: reopening flushed file at 1733750570103 (+16 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for f4be257e1d5e695afc605404060499d5 in 40ms, sequenceid=5, compaction requested=false at 1733750570111 (+8 ms)Writing region close event to WAL at 1733750570112 (+1 ms)Running coprocessor post-close hooks at 1733750570116 (+4 ms)Closed at 1733750570118 (+2 ms) 2024-12-09T13:22:50,119 INFO [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding f4be257e1d5e695afc605404060499d5 move to cad571b91d50,39675,1733750548162 record at close sequenceid=5 2024-12-09T13:22:50,122 INFO [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:50,123 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f4be257e1d5e695afc605404060499d5, regionState=CLOSED 2024-12-09T13:22:50,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure f4be257e1d5e695afc605404060499d5, server=cad571b91d50,34387,1733750548376 because future has completed 2024-12-09T13:22:50,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-09T13:22:50,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure f4be257e1d5e695afc605404060499d5, server=cad571b91d50,34387,1733750548376 in 215 msec 2024-12-09T13:22:50,131 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f4be257e1d5e695afc605404060499d5, REOPEN/MOVE; state=CLOSED, location=cad571b91d50,39675,1733750548162; forceNewPlan=false, retain=false 2024-12-09T13:22:50,281 INFO [cad571b91d50:44823 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T13:22:50,282 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f4be257e1d5e695afc605404060499d5, regionState=OPENING, regionLocation=cad571b91d50,39675,1733750548162 2024-12-09T13:22:50,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f4be257e1d5e695afc605404060499d5, REOPEN/MOVE because future has completed 2024-12-09T13:22:50,285 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure f4be257e1d5e695afc605404060499d5, server=cad571b91d50,39675,1733750548162}] 2024-12-09T13:22:50,441 INFO [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:50,441 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => f4be257e1d5e695afc605404060499d5, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5.', STARTKEY => '', ENDKEY => ''} 2024-12-09T13:22:50,442 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:50,442 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:50,442 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:50,442 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:50,444 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:50,445 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4be257e1d5e695afc605404060499d5 columnFamilyName cf1 2024-12-09T13:22:50,445 DEBUG [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:50,451 DEBUG [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf1/d1f05c7d445a43758e68ceb89e276978 2024-12-09T13:22:50,452 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.HStore(327): Store=f4be257e1d5e695afc605404060499d5/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:50,452 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:50,453 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4be257e1d5e695afc605404060499d5 columnFamilyName cf2 2024-12-09T13:22:50,453 DEBUG [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:50,453 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.HStore(327): Store=f4be257e1d5e695afc605404060499d5/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:50,454 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:50,454 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:50,456 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:50,456 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:50,456 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:50,457 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T13:22:50,458 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:50,459 INFO [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened f4be257e1d5e695afc605404060499d5; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62269794, jitterRate=-0.07210776209831238}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T13:22:50,459 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:50,460 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for f4be257e1d5e695afc605404060499d5: Running coprocessor pre-open hook at 1733750570442Writing region info on filesystem at 1733750570442Initializing all the Stores at 1733750570443 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750570443Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750570444 (+1 ms)Cleaning up temporary data from old regions at 1733750570456 (+12 ms)Running coprocessor post-open hooks at 1733750570460 (+4 ms)Region opened successfully at 1733750570460 2024-12-09T13:22:50,461 INFO [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., pid=9, masterSystemTime=1733750570437 2024-12-09T13:22:50,464 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:50,464 INFO [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:50,465 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f4be257e1d5e695afc605404060499d5, regionState=OPEN, openSeqNum=9, regionLocation=cad571b91d50,39675,1733750548162 2024-12-09T13:22:50,468 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure f4be257e1d5e695afc605404060499d5, server=cad571b91d50,39675,1733750548162 because future has completed 2024-12-09T13:22:50,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-09T13:22:50,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure f4be257e1d5e695afc605404060499d5, server=cad571b91d50,39675,1733750548162 in 184 msec 2024-12-09T13:22:50,475 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f4be257e1d5e695afc605404060499d5, REOPEN/MOVE in 569 msec 2024-12-09T13:22:50,512 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T13:22:50,514 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T13:22:50,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34387 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:50316 deadline: 1733750630519, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=cad571b91d50 port=39675 startCode=1733750548162. As of locationSeqNum=5. 2024-12-09T13:22:50,528 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,34387,1733750548376, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,34387,1733750548376, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=cad571b91d50 port=39675 startCode=1733750548162. As of locationSeqNum=5. 2024-12-09T13:22:50,529 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,34387,1733750548376, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=cad571b91d50 port=39675 startCode=1733750548162. As of locationSeqNum=5. 2024-12-09T13:22:50,529 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,34387,1733750548376, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,39675,1733750548162, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=cad571b91d50 port=39675 startCode=1733750548162. As of locationSeqNum=5. 2024-12-09T13:22:50,640 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T13:22:50,642 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T13:22:50,654 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f4be257e1d5e695afc605404060499d5 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-09T13:22:50,678 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/.tmp/cf1/e42567729a1f4c4cb2cf81aaea481cc9 is 29, key is r1/cf1:/1733750570644/DeleteFamily/seqid=0 2024-12-09T13:22:50,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741888_1066 (size=4906) 2024-12-09T13:22:50,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741888_1066 (size=4906) 2024-12-09T13:22:50,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741888_1066 (size=4906) 2024-12-09T13:22:50,687 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/.tmp/cf1/e42567729a1f4c4cb2cf81aaea481cc9 2024-12-09T13:22:50,693 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e42567729a1f4c4cb2cf81aaea481cc9 2024-12-09T13:22:50,708 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/.tmp/cf2/4cd6948f81414630a44e6b40b4d8cdce is 29, key is r1/cf2:/1733750570644/DeleteFamily/seqid=0 2024-12-09T13:22:50,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741889_1067 (size=4906) 2024-12-09T13:22:50,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741889_1067 (size=4906) 2024-12-09T13:22:50,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741889_1067 (size=4906) 2024-12-09T13:22:50,715 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/.tmp/cf2/4cd6948f81414630a44e6b40b4d8cdce 2024-12-09T13:22:50,721 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4cd6948f81414630a44e6b40b4d8cdce 2024-12-09T13:22:50,722 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/.tmp/cf1/e42567729a1f4c4cb2cf81aaea481cc9 as hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf1/e42567729a1f4c4cb2cf81aaea481cc9 2024-12-09T13:22:50,728 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e42567729a1f4c4cb2cf81aaea481cc9 2024-12-09T13:22:50,728 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf1/e42567729a1f4c4cb2cf81aaea481cc9, entries=1, sequenceid=12, filesize=4.8 K 2024-12-09T13:22:50,729 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/.tmp/cf2/4cd6948f81414630a44e6b40b4d8cdce as hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf2/4cd6948f81414630a44e6b40b4d8cdce 2024-12-09T13:22:50,735 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4cd6948f81414630a44e6b40b4d8cdce 2024-12-09T13:22:50,735 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf2/4cd6948f81414630a44e6b40b4d8cdce, entries=1, sequenceid=12, filesize=4.8 K 2024-12-09T13:22:50,736 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for f4be257e1d5e695afc605404060499d5 in 82ms, sequenceid=12, compaction requested=false 2024-12-09T13:22:50,737 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f4be257e1d5e695afc605404060499d5: 2024-12-09T13:22:50,740 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T13:22:50,741 DEBUG [Time-limited test {}] regionserver.HStore(1541): f4be257e1d5e695afc605404060499d5/cf1 is initiating major compaction (all files) 2024-12-09T13:22:50,741 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T13:22:50,741 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:50,742 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of f4be257e1d5e695afc605404060499d5/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:50,742 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf1/d1f05c7d445a43758e68ceb89e276978, hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf1/e42567729a1f4c4cb2cf81aaea481cc9] into tmpdir=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/.tmp, totalSize=9.5 K 2024-12-09T13:22:50,743 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d1f05c7d445a43758e68ceb89e276978, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733750569885 2024-12-09T13:22:50,744 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e42567729a1f4c4cb2cf81aaea481cc9, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-09T13:22:50,754 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): f4be257e1d5e695afc605404060499d5#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T13:22:50,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741890_1068 (size=4626) 2024-12-09T13:22:50,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741890_1068 (size=4626) 2024-12-09T13:22:50,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741890_1068 (size=4626) 2024-12-09T13:22:50,766 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/.tmp/cf1/3e04f215452643cc9b4791c8c6999936 as hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf1/3e04f215452643cc9b4791c8c6999936 2024-12-09T13:22:50,780 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in f4be257e1d5e695afc605404060499d5/cf1 of f4be257e1d5e695afc605404060499d5 into 3e04f215452643cc9b4791c8c6999936(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T13:22:50,781 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for f4be257e1d5e695afc605404060499d5: 2024-12-09T13:22:50,781 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-09T13:22:50,781 DEBUG [Time-limited test {}] regionserver.HStore(1541): f4be257e1d5e695afc605404060499d5/cf2 is initiating major compaction (all files) 2024-12-09T13:22:50,781 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T13:22:50,781 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T13:22:50,781 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of f4be257e1d5e695afc605404060499d5/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:50,781 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf2/4cd6948f81414630a44e6b40b4d8cdce] into tmpdir=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/.tmp, totalSize=4.8 K 2024-12-09T13:22:50,782 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 4cd6948f81414630a44e6b40b4d8cdce, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-09T13:22:50,788 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): f4be257e1d5e695afc605404060499d5#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T13:22:50,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741891_1069 (size=4592) 2024-12-09T13:22:50,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741891_1069 (size=4592) 2024-12-09T13:22:50,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741891_1069 (size=4592) 2024-12-09T13:22:50,803 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/.tmp/cf2/cd51c8e820424368bde34ae76d8c3a45 as hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf2/cd51c8e820424368bde34ae76d8c3a45 2024-12-09T13:22:50,809 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in f4be257e1d5e695afc605404060499d5/cf2 of f4be257e1d5e695afc605404060499d5 into cd51c8e820424368bde34ae76d8c3a45(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T13:22:50,809 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for f4be257e1d5e695afc605404060499d5: 2024-12-09T13:22:50,814 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44823 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=f4be257e1d5e695afc605404060499d5, source=cad571b91d50,39675,1733750548162, destination=cad571b91d50,34387,1733750548376, warming up region on cad571b91d50,34387,1733750548376 2024-12-09T13:22:50,814 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44823 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=f4be257e1d5e695afc605404060499d5, source=cad571b91d50,39675,1733750548162, destination=cad571b91d50,34387,1733750548376, running balancer 2024-12-09T13:22:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44823 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f4be257e1d5e695afc605404060499d5, REOPEN/MOVE 2024-12-09T13:22:50,816 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f4be257e1d5e695afc605404060499d5, REOPEN/MOVE 2024-12-09T13:22:50,817 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34387 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34387 {}] regionserver.HRegion(7855): Warmup {ENCODED => f4be257e1d5e695afc605404060499d5, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5.', STARTKEY => '', ENDKEY => ''} 2024-12-09T13:22:50,817 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f4be257e1d5e695afc605404060499d5, regionState=CLOSING, regionLocation=cad571b91d50,39675,1733750548162 2024-12-09T13:22:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34387 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:50,818 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:50,819 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4be257e1d5e695afc605404060499d5 columnFamilyName cf1 2024-12-09T13:22:50,819 DEBUG [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:50,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f4be257e1d5e695afc605404060499d5, REOPEN/MOVE because future has completed 2024-12-09T13:22:50,820 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T13:22:50,820 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure f4be257e1d5e695afc605404060499d5, server=cad571b91d50,39675,1733750548162}] 2024-12-09T13:22:50,827 DEBUG [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf1/3e04f215452643cc9b4791c8c6999936 2024-12-09T13:22:50,832 DEBUG [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf1/d1f05c7d445a43758e68ceb89e276978 2024-12-09T13:22:50,837 INFO [StoreFileOpener-f4be257e1d5e695afc605404060499d5-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e42567729a1f4c4cb2cf81aaea481cc9 2024-12-09T13:22:50,837 DEBUG [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf1/e42567729a1f4c4cb2cf81aaea481cc9 2024-12-09T13:22:50,837 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.HStore(327): Store=f4be257e1d5e695afc605404060499d5/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:50,837 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:50,838 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4be257e1d5e695afc605404060499d5 columnFamilyName cf2 2024-12-09T13:22:50,838 DEBUG [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:50,845 INFO [StoreFileOpener-f4be257e1d5e695afc605404060499d5-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4cd6948f81414630a44e6b40b4d8cdce 2024-12-09T13:22:50,845 DEBUG [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf2/4cd6948f81414630a44e6b40b4d8cdce 2024-12-09T13:22:50,850 DEBUG [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf2/cd51c8e820424368bde34ae76d8c3a45 2024-12-09T13:22:50,850 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.HStore(327): Store=f4be257e1d5e695afc605404060499d5/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34387 {}] regionserver.HRegion(1722): Closing f4be257e1d5e695afc605404060499d5, disabling compactions & flushes 2024-12-09T13:22:50,850 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34387 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34387 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34387 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. after waiting 0 ms 2024-12-09T13:22:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34387 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:50,855 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34387 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34387 {}] regionserver.HRegion(1676): Region close journal for f4be257e1d5e695afc605404060499d5: Waiting for close lock at 1733750570850Disabling compacts and flushes for region at 1733750570850Disabling writes for close at 1733750570850Writing region close event to WAL at 1733750570855 (+5 ms)Closed at 1733750570855 2024-12-09T13:22:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44823 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-09T13:22:50,974 INFO [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:50,975 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T13:22:50,975 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing f4be257e1d5e695afc605404060499d5, disabling compactions & flushes 2024-12-09T13:22:50,975 INFO [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:50,975 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:50,975 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. after waiting 0 ms 2024-12-09T13:22:50,975 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:50,975 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf1/d1f05c7d445a43758e68ceb89e276978, hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf1/e42567729a1f4c4cb2cf81aaea481cc9] to archive 2024-12-09T13:22:50,978 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T13:22:50,983 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf1/d1f05c7d445a43758e68ceb89e276978 to hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf1/d1f05c7d445a43758e68ceb89e276978 2024-12-09T13:22:50,985 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf1/e42567729a1f4c4cb2cf81aaea481cc9 to hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf1/e42567729a1f4c4cb2cf81aaea481cc9 2024-12-09T13:22:50,997 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf2/4cd6948f81414630a44e6b40b4d8cdce] to archive 2024-12-09T13:22:50,999 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T13:22:51,001 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf2/4cd6948f81414630a44e6b40b4d8cdce to hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf2/4cd6948f81414630a44e6b40b4d8cdce 2024-12-09T13:22:51,006 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-09T13:22:51,007 INFO [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:51,007 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for f4be257e1d5e695afc605404060499d5: Waiting for close lock at 1733750570975Running coprocessor pre-close hooks at 1733750570975Disabling compacts and flushes for region at 1733750570975Disabling writes for close at 1733750570975Writing region close event to WAL at 1733750571002 (+27 ms)Running coprocessor post-close hooks at 1733750571007 (+5 ms)Closed at 1733750571007 2024-12-09T13:22:51,007 INFO [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding f4be257e1d5e695afc605404060499d5 move to cad571b91d50,34387,1733750548376 record at close sequenceid=12 2024-12-09T13:22:51,010 INFO [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:51,011 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f4be257e1d5e695afc605404060499d5, regionState=CLOSED 2024-12-09T13:22:51,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure f4be257e1d5e695afc605404060499d5, server=cad571b91d50,39675,1733750548162 because future has completed 2024-12-09T13:22:51,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-09T13:22:51,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure f4be257e1d5e695afc605404060499d5, server=cad571b91d50,39675,1733750548162 in 197 msec 2024-12-09T13:22:51,021 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f4be257e1d5e695afc605404060499d5, REOPEN/MOVE; state=CLOSED, location=cad571b91d50,34387,1733750548376; forceNewPlan=false, retain=false 2024-12-09T13:22:51,172 INFO [cad571b91d50:44823 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T13:22:51,172 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f4be257e1d5e695afc605404060499d5, regionState=OPENING, regionLocation=cad571b91d50,34387,1733750548376 2024-12-09T13:22:51,175 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f4be257e1d5e695afc605404060499d5, REOPEN/MOVE because future has completed 2024-12-09T13:22:51,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure f4be257e1d5e695afc605404060499d5, server=cad571b91d50,34387,1733750548376}] 2024-12-09T13:22:51,332 INFO [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:51,333 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => f4be257e1d5e695afc605404060499d5, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5.', STARTKEY => '', ENDKEY => ''} 2024-12-09T13:22:51,333 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:51,333 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:51,333 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:51,333 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:51,335 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:51,336 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4be257e1d5e695afc605404060499d5 columnFamilyName cf1 2024-12-09T13:22:51,336 DEBUG [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:51,341 DEBUG [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf1/3e04f215452643cc9b4791c8c6999936 2024-12-09T13:22:51,342 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.HStore(327): Store=f4be257e1d5e695afc605404060499d5/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:51,342 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:51,343 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4be257e1d5e695afc605404060499d5 columnFamilyName cf2 2024-12-09T13:22:51,343 DEBUG [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:51,348 DEBUG [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf2/cd51c8e820424368bde34ae76d8c3a45 2024-12-09T13:22:51,348 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.HStore(327): Store=f4be257e1d5e695afc605404060499d5/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:51,348 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:51,349 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:51,350 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:51,351 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:51,351 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:51,352 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T13:22:51,353 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:51,354 INFO [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened f4be257e1d5e695afc605404060499d5; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73117345, jitterRate=0.08953334391117096}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T13:22:51,354 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:51,354 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for f4be257e1d5e695afc605404060499d5: Running coprocessor pre-open hook at 1733750571333Writing region info on filesystem at 1733750571333Initializing all the Stores at 1733750571334 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750571334Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750571335 (+1 ms)Cleaning up temporary data from old regions at 1733750571351 (+16 ms)Running coprocessor post-open hooks at 1733750571354 (+3 ms)Region opened successfully at 1733750571354 2024-12-09T13:22:51,355 INFO [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., pid=12, masterSystemTime=1733750571328 2024-12-09T13:22:51,358 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:51,358 INFO [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:51,359 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f4be257e1d5e695afc605404060499d5, regionState=OPEN, openSeqNum=18, regionLocation=cad571b91d50,34387,1733750548376 2024-12-09T13:22:51,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure f4be257e1d5e695afc605404060499d5, server=cad571b91d50,34387,1733750548376 because future has completed 2024-12-09T13:22:51,365 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-09T13:22:51,365 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure f4be257e1d5e695afc605404060499d5, server=cad571b91d50,34387,1733750548376 in 187 msec 2024-12-09T13:22:51,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f4be257e1d5e695afc605404060499d5, REOPEN/MOVE in 551 msec 2024-12-09T13:22:51,418 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T13:22:51,420 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T13:22:51,421 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server cad571b91d50,34387,1733750548376: testing ***** 2024-12-09T13:22:51,422 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-09T13:22:51,425 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-09T13:22:51,426 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-09T13:22:51,432 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-09T13:22:51,434 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-09T13:22:51,444 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 492744160 }, "NonHeapMemoryUsage": { "committed": 170917888, "init": 7667712, "max": -1, "used": 168427272 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "cad571b91d50", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 11076, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 40, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 3, "ProcessCallTime_90th_percentile": 6, "ProcessCallTime_95th_percentile": 9, "ProcessCallTime_98th_percentile": 40, "ProcessCallTime_99th_percentile": 40, "ProcessCallTime_99.9th_percentile": 40, "ProcessCallTime_TimeRangeCount_0-1": 12, "ProcessCallTime_TimeRangeCount_1-3": 12, "ProcessCallTime_TimeRangeCount_3-10": 6, "ProcessCallTime_TimeRangeCount_30-100": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 40, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 3, "TotalCallTime_90th_percentile": 7, "TotalCallTime_95th_percentile": 9, "TotalCallTime_98th_percentile": 40, "TotalCallTime_99th_percentile": 40, "TotalCallTime_99.9th_percentile": 40, "TotalCallTime_TimeRangeCount_0-1": 12, "TotalCallTime_TimeRangeCount_1-3": 12, "TotalCallTime_TimeRangeCount_3-10": 6, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1139, "ResponseSize_mean": 159, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 796, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1070, "ResponseSize_99.9th_percentile": 1132, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 5, "ResponseSize_SizeRangeCount_100-1000": 4, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5752 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "cad571b91d50", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:53371", "tag.serverName": "cad571b91d50,34387,1733750548376", "tag.clusterId": "08bed1e9-22ca-42c0-b4c7-615143020766", "tag.Context": "regionserver", "tag.Hostname": "cad571b91d50", "regionCount": 1, "storeCount": 0, "hlogFileCount": 4, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733750548376, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227243, "ByteBuffAllocatorPoolAllocationBytes": 66560, "ByteBuffAllocatorHeapAllocationRatio": 0.7697053155307054, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 4, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 1, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 1, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 1, "Put_min": 2, "Put_max": 2, "Put_mean": 2, "Put_25th_percentile": 2, "Put_median": 2, "Put_75th_percentile": 2, "Put_90th_percentile": 2, "Put_95th_percentile": 2, "Put_98th_percentile": 2, "Put_99th_percentile": 2, "Put_99.9th_percentile": 2, "Put_TimeRangeCount_1-3": 1, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 4783, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 31, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 4783, "FlushOutputSize_max": 4783, "FlushOutputSize_mean": 4783, "FlushOutputSize_25th_percentile": 4783, "FlushOutputSize_median": 4783, "FlushOutputSize_75th_percentile": 4783, "FlushOutputSize_90th_percentile": 4783, "FlushOutputSize_95th_percentile": 4783, "FlushOutputSize_98th_percentile": 4783, "FlushOutputSize_99th_percentile": 4783, "FlushOutputSize_99.9th_percentile": 4783, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 31, "FlushMemstoreSize_max": 31, "FlushMemstoreSize_mean": 31, "FlushMemstoreSize_25th_percentile": 31, "FlushMemstoreSize_median": 31, "FlushMemstoreSize_75th_percentile": 31, "FlushMemstoreSize_90th_percentile": 31, "FlushMemstoreSize_95th_percentile": 31, "FlushMemstoreSize_98th_percentile": 31, "FlushMemstoreSize_99th_percentile": 31, "FlushMemstoreSize_99.9th_percentile": 31, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 40, "FlushTime_max": 40, "FlushTime_mean": 40, "FlushTime_25th_percentile": 40, "FlushTime_median": 40, "FlushTime_75th_percentile": 40, "FlushTime_90th_percentile": 40, "FlushTime_95th_percentile": 40, "FlushTime_98th_percentile": 40, "FlushTime_99th_percentile": 40, "FlushTime_99.9th_percentile": 40, "FlushTime_TimeRangeCount_30-100": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 1, "ScanSize_min": 64, "ScanSize_max": 64, "ScanSize_mean": 64, "ScanSize_25th_percentile": 64, "ScanSize_median": 64, "ScanSize_75th_percentile": 64, "ScanSize_90th_percentile": 64, "ScanSize_95th_percentile": 64, "ScanSize_98th_percentile": 64, "ScanSize_99th_percentile": 64, "ScanSize_99.9th_percentile": 64, "ScanSize_SizeRangeCount_10-100": 1, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-09T13:22:51,448 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44823 {}] master.MasterRpcServices(700): cad571b91d50,34387,1733750548376 reported a fatal error: ***** ABORTING region server cad571b91d50,34387,1733750548376: testing ***** 2024-12-09T13:22:51,450 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'cad571b91d50,34387,1733750548376' ***** 2024-12-09T13:22:51,450 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-09T13:22:51,450 INFO [RS:1;cad571b91d50:34387 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T13:22:51,451 INFO [RS:1;cad571b91d50:34387 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-09T13:22:51,451 INFO [RS:1;cad571b91d50:34387 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-09T13:22:51,451 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T13:22:51,451 INFO [RS:1;cad571b91d50:34387 {}] regionserver.HRegionServer(3091): Received CLOSE for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:51,451 INFO [RS:1;cad571b91d50:34387 {}] regionserver.HRegionServer(956): aborting server cad571b91d50,34387,1733750548376 2024-12-09T13:22:51,452 INFO [RS:1;cad571b91d50:34387 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T13:22:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39675 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:54320 deadline: 1733750631451, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=cad571b91d50 port=34387 startCode=1733750548376. As of locationSeqNum=12. 2024-12-09T13:22:51,452 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f4be257e1d5e695afc605404060499d5, disabling compactions & flushes 2024-12-09T13:22:51,452 INFO [RS:1;cad571b91d50:34387 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;cad571b91d50:34387. 2024-12-09T13:22:51,452 INFO [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:51,452 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:51,452 DEBUG [RS:1;cad571b91d50:34387 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T13:22:51,452 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. after waiting 0 ms 2024-12-09T13:22:51,452 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:51,452 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,39675,1733750548162, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,39675,1733750548162, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=cad571b91d50 port=34387 startCode=1733750548376. As of locationSeqNum=12. 2024-12-09T13:22:51,452 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,39675,1733750548162, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=cad571b91d50 port=34387 startCode=1733750548376. As of locationSeqNum=12. 2024-12-09T13:22:51,452 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,39675,1733750548162, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,34387,1733750548376, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=cad571b91d50 port=34387 startCode=1733750548376. As of locationSeqNum=12. 2024-12-09T13:22:51,453 DEBUG [RS:1;cad571b91d50:34387 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T13:22:51,454 INFO [RS:1;cad571b91d50:34387 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T13:22:51,454 INFO [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:51,454 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f4be257e1d5e695afc605404060499d5: Waiting for close lock at 1733750571451Running coprocessor pre-close hooks at 1733750571452 (+1 ms)Disabling compacts and flushes for region at 1733750571452Disabling writes for close at 1733750571452Writing region close event to WAL at 1733750571453 (+1 ms)Running coprocessor post-close hooks at 1733750571454 (+1 ms)Closed at 1733750571454 2024-12-09T13:22:51,454 DEBUG [RS:1;cad571b91d50:34387 {}] regionserver.HRegionServer(1325): Online Regions={f4be257e1d5e695afc605404060499d5=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5.} 2024-12-09T13:22:51,454 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:51,455 DEBUG [RS:1;cad571b91d50:34387 {}] regionserver.HRegionServer(1351): Waiting on 2024-12-09T13:22:51,480 INFO [regionserver/cad571b91d50:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-09T13:22:51,480 INFO [regionserver/cad571b91d50:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-09T13:22:51,495 INFO [regionserver/cad571b91d50:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T13:22:51,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server cad571b91d50,34387,1733750548376 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:22:51,560 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,34387,1733750548376, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,34387,1733750548376, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server cad571b91d50,34387,1733750548376 aborting 2024-12-09T13:22:51,560 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,34387,1733750548376, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server cad571b91d50,34387,1733750548376 aborting 2024-12-09T13:22:51,560 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,34387,1733750548376, seqNum=12 from cache 2024-12-09T13:22:51,655 INFO [RS:1;cad571b91d50:34387 {}] regionserver.HRegionServer(976): stopping server cad571b91d50,34387,1733750548376; all regions closed. 2024-12-09T13:22:51,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741833_1009 (size=2142) 2024-12-09T13:22:51,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741833_1009 (size=2142) 2024-12-09T13:22:51,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741833_1009 (size=2142) 2024-12-09T13:22:51,660 DEBUG [RS:1;cad571b91d50:34387 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T13:22:51,660 INFO [RS:1;cad571b91d50:34387 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T13:22:51,660 INFO [RS:1;cad571b91d50:34387 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T13:22:51,661 INFO [RS:1;cad571b91d50:34387 {}] hbase.ChoreService(370): Chore service for: regionserver/cad571b91d50:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T13:22:51,661 INFO [regionserver/cad571b91d50:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T13:22:51,661 INFO [RS:1;cad571b91d50:34387 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T13:22:51,661 INFO [RS:1;cad571b91d50:34387 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T13:22:51,661 INFO [RS:1;cad571b91d50:34387 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T13:22:51,661 INFO [RS:1;cad571b91d50:34387 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T13:22:51,662 INFO [RS:1;cad571b91d50:34387 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34387 2024-12-09T13:22:51,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34387-0x1012b596d9a0002, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/cad571b91d50,34387,1733750548376 2024-12-09T13:22:51,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T13:22:51,670 INFO [RS:1;cad571b91d50:34387 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T13:22:51,672 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [cad571b91d50,34387,1733750548376] 2024-12-09T13:22:51,674 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/cad571b91d50,34387,1733750548376 already deleted, retry=false 2024-12-09T13:22:51,674 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of cad571b91d50,34387,1733750548376 on cad571b91d50,44823,1733750546819 2024-12-09T13:22:51,679 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure cad571b91d50,34387,1733750548376, splitWal=true, meta=false 2024-12-09T13:22:51,682 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for cad571b91d50,34387,1733750548376 (carryingMeta=false) cad571b91d50,34387,1733750548376/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@227fb7be[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-09T13:22:51,682 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure cad571b91d50,34387,1733750548376, splitWal=true, meta=false 2024-12-09T13:22:51,684 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(207): cad571b91d50,34387,1733750548376 had 1 regions 2024-12-09T13:22:51,685 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure cad571b91d50,34387,1733750548376, splitWal=true, meta=false, isMeta: false 2024-12-09T13:22:51,686 DEBUG [PEWorker-5 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34387,1733750548376-splitting 2024-12-09T13:22:51,687 INFO [PEWorker-5 {}] master.SplitWALManager(105): cad571b91d50,34387,1733750548376 WAL count=1, meta=false 2024-12-09T13:22:51,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure cad571b91d50%2C34387%2C1733750548376.1733750551798}] 2024-12-09T13:22:51,694 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=cad571b91d50,34317,1733750548453 2024-12-09T13:22:51,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure cad571b91d50%2C34387%2C1733750548376.1733750551798, worker=cad571b91d50,34317,1733750548453}] 2024-12-09T13:22:51,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34387-0x1012b596d9a0002, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T13:22:51,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34387-0x1012b596d9a0002, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T13:22:51,773 INFO [RS:1;cad571b91d50:34387 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T13:22:51,773 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,34387,1733750548376, seqNum=18] 2024-12-09T13:22:51,773 INFO [RS:1;cad571b91d50:34387 {}] regionserver.HRegionServer(1031): Exiting; stopping=cad571b91d50,34387,1733750548376; zookeeper connection closed. 2024-12-09T13:22:51,774 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@320881ea {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@320881ea 2024-12-09T13:22:51,775 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server cad571b91d50:34387 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: cad571b91d50/172.17.0.3:34387 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:22:51,776 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,34387,1733750548376, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,34387,1733750548376, seqNum=18, error=java.net.ConnectException: Call to address=cad571b91d50:34387 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: cad571b91d50/172.17.0.3:34387 2024-12-09T13:22:51,776 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,34387,1733750548376, seqNum=18 is java.net.ConnectException: Connection refused 2024-12-09T13:22:51,776 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,34387,1733750548376, seqNum=18 from cache 2024-12-09T13:22:51,777 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address cad571b91d50:34387 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: cad571b91d50/172.17.0.3:34387 2024-12-09T13:22:51,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34317 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-09T13:22:51,879 INFO [RS_LOG_REPLAY_OPS-regionserver/cad571b91d50:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34387,1733750548376-splitting/cad571b91d50%2C34387%2C1733750548376.1733750551798, size=2.1 K (2142bytes) 2024-12-09T13:22:51,880 INFO [RS_LOG_REPLAY_OPS-regionserver/cad571b91d50:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34387,1733750548376-splitting/cad571b91d50%2C34387%2C1733750548376.1733750551798 2024-12-09T13:22:51,880 INFO [RS_LOG_REPLAY_OPS-regionserver/cad571b91d50:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34387,1733750548376-splitting/cad571b91d50%2C34387%2C1733750548376.1733750551798 after 0ms 2024-12-09T13:22:51,883 INFO [RS_LOG_REPLAY_OPS-regionserver/cad571b91d50:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34387,1733750548376-splitting/cad571b91d50%2C34387%2C1733750548376.1733750551798 took 4ms 2024-12-09T13:22:51,890 DEBUG [RS_LOG_REPLAY_OPS-regionserver/cad571b91d50:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for f4be257e1d5e695afc605404060499d5: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-09T13:22:51,890 DEBUG [RS_LOG_REPLAY_OPS-regionserver/cad571b91d50:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34387,1733750548376-splitting/cad571b91d50%2C34387%2C1733750548376.1733750551798 so closing down 2024-12-09T13:22:51,890 DEBUG [RS_LOG_REPLAY_OPS-regionserver/cad571b91d50:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T13:22:51,890 INFO [RS_LOG_REPLAY_OPS-regionserver/cad571b91d50:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T13:22:51,891 INFO [RS_LOG_REPLAY_OPS-regionserver/cad571b91d50:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 8 ms; skipped=6; WAL=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34387,1733750548376-splitting/cad571b91d50%2C34387%2C1733750548376.1733750551798, size=2.1 K, length=2142, corrupted=false, cancelled=false 2024-12-09T13:22:51,891 DEBUG [RS_LOG_REPLAY_OPS-regionserver/cad571b91d50:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34387,1733750548376-splitting/cad571b91d50%2C34387%2C1733750548376.1733750551798, journal: Splitting hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34387,1733750548376-splitting/cad571b91d50%2C34387%2C1733750548376.1733750551798, size=2.1 K (2142bytes) at 1733750571879Finishing writing output for hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34387,1733750548376-splitting/cad571b91d50%2C34387%2C1733750548376.1733750551798 so closing down at 1733750571890 (+11 ms)3 split writer threads finished at 1733750571891 (+1 ms)Processed 6 edits across 0 Regions in 8 ms; skipped=6; WAL=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34387,1733750548376-splitting/cad571b91d50%2C34387%2C1733750548376.1733750551798, size=2.1 K, length=2142, corrupted=false, cancelled=false at 1733750571891 2024-12-09T13:22:51,891 DEBUG [RS_LOG_REPLAY_OPS-regionserver/cad571b91d50:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34387,1733750548376-splitting/cad571b91d50%2C34387%2C1733750548376.1733750551798 2024-12-09T13:22:51,892 DEBUG [RS_LOG_REPLAY_OPS-regionserver/cad571b91d50:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-09T13:22:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44823 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-09T13:22:51,899 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34387,1733750548376-splitting/cad571b91d50%2C34387%2C1733750548376.1733750551798 to hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/oldWALs 2024-12-09T13:22:51,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-09T13:22:51,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure cad571b91d50%2C34387%2C1733750548376.1733750551798, worker=cad571b91d50,34317,1733750548453 in 202 msec 2024-12-09T13:22:51,905 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=cad571b91d50,34317,1733750548453 2024-12-09T13:22:51,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-09T13:22:51,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure cad571b91d50%2C34387%2C1733750548376.1733750551798, worker=cad571b91d50,34317,1733750548453 in 216 msec 2024-12-09T13:22:51,909 INFO [PEWorker-5 {}] master.SplitLogManager(171): hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34387,1733750548376-splitting dir is empty, no logs to split. 2024-12-09T13:22:51,910 INFO [PEWorker-5 {}] master.SplitWALManager(105): cad571b91d50,34387,1733750548376 WAL count=0, meta=false 2024-12-09T13:22:51,910 DEBUG [PEWorker-5 {}] procedure.ServerCrashProcedure(329): Check if cad571b91d50,34387,1733750548376 WAL splitting is done? wals=0, meta=false 2024-12-09T13:22:51,912 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for cad571b91d50,34387,1733750548376 failed, ignore...File hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/WALs/cad571b91d50,34387,1733750548376-splitting does not exist. 2024-12-09T13:22:51,913 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f4be257e1d5e695afc605404060499d5, ASSIGN}] 2024-12-09T13:22:51,915 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f4be257e1d5e695afc605404060499d5, ASSIGN 2024-12-09T13:22:51,915 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f4be257e1d5e695afc605404060499d5, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-09T13:22:52,066 DEBUG [cad571b91d50:44823 {}] balancer.BalancerClusterState(204): Hosts are {cad571b91d50=0} racks are {/default-rack=0} 2024-12-09T13:22:52,066 DEBUG [cad571b91d50:44823 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T13:22:52,066 DEBUG [cad571b91d50:44823 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T13:22:52,066 DEBUG [cad571b91d50:44823 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T13:22:52,066 DEBUG [cad571b91d50:44823 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T13:22:52,066 INFO [cad571b91d50:44823 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T13:22:52,066 INFO [cad571b91d50:44823 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T13:22:52,066 DEBUG [cad571b91d50:44823 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T13:22:52,067 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=f4be257e1d5e695afc605404060499d5, regionState=OPENING, regionLocation=cad571b91d50,39675,1733750548162 2024-12-09T13:22:52,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f4be257e1d5e695afc605404060499d5, ASSIGN because future has completed 2024-12-09T13:22:52,070 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure f4be257e1d5e695afc605404060499d5, server=cad571b91d50,39675,1733750548162}] 2024-12-09T13:22:52,082 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,34387,1733750548376, seqNum=18] 2024-12-09T13:22:52,082 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to cad571b91d50:34387 this server is in the failed servers list 2024-12-09T13:22:52,083 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,34387,1733750548376, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,34387,1733750548376, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=cad571b91d50:34387 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: cad571b91d50:34387 2024-12-09T13:22:52,083 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,34387,1733750548376, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: cad571b91d50:34387 2024-12-09T13:22:52,083 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,34387,1733750548376, seqNum=18 from cache 2024-12-09T13:22:52,226 INFO [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:52,227 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => f4be257e1d5e695afc605404060499d5, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5.', STARTKEY => '', ENDKEY => ''} 2024-12-09T13:22:52,227 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:52,227 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:52,227 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:52,227 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:52,229 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:52,230 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4be257e1d5e695afc605404060499d5 columnFamilyName cf1 2024-12-09T13:22:52,230 DEBUG [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:52,236 DEBUG [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf1/3e04f215452643cc9b4791c8c6999936 2024-12-09T13:22:52,237 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.HStore(327): Store=f4be257e1d5e695afc605404060499d5/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:52,237 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:52,239 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4be257e1d5e695afc605404060499d5 columnFamilyName cf2 2024-12-09T13:22:52,239 DEBUG [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:52,249 DEBUG [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/cf2/cd51c8e820424368bde34ae76d8c3a45 2024-12-09T13:22:52,249 INFO [StoreOpener-f4be257e1d5e695afc605404060499d5-1 {}] regionserver.HStore(327): Store=f4be257e1d5e695afc605404060499d5/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:52,250 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:52,250 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:52,252 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:52,252 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:52,252 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:52,253 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T13:22:52,254 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:52,255 INFO [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened f4be257e1d5e695afc605404060499d5; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65700379, jitterRate=-0.020988062024116516}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T13:22:52,256 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f4be257e1d5e695afc605404060499d5 2024-12-09T13:22:52,256 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for f4be257e1d5e695afc605404060499d5: Running coprocessor pre-open hook at 1733750572228Writing region info on filesystem at 1733750572228Initializing all the Stores at 1733750572228Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750572228Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750572229 (+1 ms)Cleaning up temporary data from old regions at 1733750572252 (+23 ms)Running coprocessor post-open hooks at 1733750572256 (+4 ms)Region opened successfully at 1733750572256 2024-12-09T13:22:52,257 INFO [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., pid=17, masterSystemTime=1733750572222 2024-12-09T13:22:52,260 DEBUG [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:52,260 INFO [RS_OPEN_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:22:52,261 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=f4be257e1d5e695afc605404060499d5, regionState=OPEN, openSeqNum=18, regionLocation=cad571b91d50,39675,1733750548162 2024-12-09T13:22:52,263 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure f4be257e1d5e695afc605404060499d5, server=cad571b91d50,39675,1733750548162 because future has completed 2024-12-09T13:22:52,270 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-09T13:22:52,271 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure f4be257e1d5e695afc605404060499d5, server=cad571b91d50,39675,1733750548162 in 197 msec 2024-12-09T13:22:52,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-09T13:22:52,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f4be257e1d5e695afc605404060499d5, ASSIGN in 358 msec 2024-12-09T13:22:52,273 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server cad571b91d50,34387,1733750548376 after splitting done 2024-12-09T13:22:52,275 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure cad571b91d50,34387,1733750548376, splitWal=true, meta=false in 599 msec 2024-12-09T13:22:52,592 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5., hostname=cad571b91d50,39675,1733750548162, seqNum=18] 2024-12-09T13:22:52,607 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=403 (was 408), OpenFileDescriptor=1048 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=724 (was 727), ProcessCount=11 (was 11), AvailableMemoryMB=3573 (was 3658) 2024-12-09T13:22:52,608 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-12-09T13:22:52,620 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=403, OpenFileDescriptor=1048, MaxFileDescriptor=1048576, SystemLoadAverage=724, ProcessCount=11, AvailableMemoryMB=3573 2024-12-09T13:22:52,620 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-12-09T13:22:52,636 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:22:52,637 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:22:52,638 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T13:22:52,640 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-24479572, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/hregion-24479572, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:22:52,654 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-24479572/hregion-24479572.1733750572641, exclude list is [], retry=0 2024-12-09T13:22:52,657 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:22:52,657 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:22:52,657 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:22:52,659 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-24479572/hregion-24479572.1733750572641 2024-12-09T13:22:52,660 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34135:34135),(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34323:34323)] 2024-12-09T13:22:52,660 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 7515d0d2e8096021274a3284a86db2a6, NAME => 'testReplayEditsWrittenViaHRegion,,1733750572636.7515d0d2e8096021274a3284a86db2a6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41625/hbase 2024-12-09T13:22:52,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741893_1071 (size=67) 2024-12-09T13:22:52,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741893_1071 (size=67) 2024-12-09T13:22:52,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741893_1071 (size=67) 2024-12-09T13:22:52,670 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733750572636.7515d0d2e8096021274a3284a86db2a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:52,672 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:52,673 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7515d0d2e8096021274a3284a86db2a6 columnFamilyName a 2024-12-09T13:22:52,673 DEBUG [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:52,674 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] regionserver.HStore(327): Store=7515d0d2e8096021274a3284a86db2a6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:52,674 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:52,675 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7515d0d2e8096021274a3284a86db2a6 columnFamilyName b 2024-12-09T13:22:52,675 DEBUG [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:52,675 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] regionserver.HStore(327): Store=7515d0d2e8096021274a3284a86db2a6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:52,676 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:52,677 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7515d0d2e8096021274a3284a86db2a6 columnFamilyName c 2024-12-09T13:22:52,677 DEBUG [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:52,677 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] regionserver.HStore(327): Store=7515d0d2e8096021274a3284a86db2a6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:52,678 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:52,678 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:52,679 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:52,680 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:52,680 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:52,680 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T13:22:52,682 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:52,684 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T13:22:52,685 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7515d0d2e8096021274a3284a86db2a6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59325164, jitterRate=-0.115986168384552}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T13:22:52,686 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7515d0d2e8096021274a3284a86db2a6: Writing region info on filesystem at 1733750572671Initializing all the Stores at 1733750572671Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750572671Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750572671Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750572671Cleaning up temporary data from old regions at 1733750572680 (+9 ms)Region opened successfully at 1733750572686 (+6 ms) 2024-12-09T13:22:52,686 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7515d0d2e8096021274a3284a86db2a6, disabling compactions & flushes 2024-12-09T13:22:52,686 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733750572636.7515d0d2e8096021274a3284a86db2a6. 2024-12-09T13:22:52,686 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733750572636.7515d0d2e8096021274a3284a86db2a6. 2024-12-09T13:22:52,686 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733750572636.7515d0d2e8096021274a3284a86db2a6. after waiting 0 ms 2024-12-09T13:22:52,686 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733750572636.7515d0d2e8096021274a3284a86db2a6. 2024-12-09T13:22:52,687 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733750572636.7515d0d2e8096021274a3284a86db2a6. 2024-12-09T13:22:52,687 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7515d0d2e8096021274a3284a86db2a6: Waiting for close lock at 1733750572686Disabling compacts and flushes for region at 1733750572686Disabling writes for close at 1733750572686Writing region close event to WAL at 1733750572687 (+1 ms)Closed at 1733750572687 2024-12-09T13:22:52,690 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-24479572/hregion-24479572.1733750572641 not finished, retry = 0 2024-12-09T13:22:52,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741892_1070 (size=93) 2024-12-09T13:22:52,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741892_1070 (size=93) 2024-12-09T13:22:52,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741892_1070 (size=93) 2024-12-09T13:22:52,793 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T13:22:52,793 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-24479572:(num 1733750572641) 2024-12-09T13:22:52,793 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T13:22:52,795 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733750572635, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:22:52,809 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733750572635/wal.1733750572795, exclude list is [], retry=0 2024-12-09T13:22:52,812 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:22:52,813 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:22:52,813 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:22:52,815 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733750572635/wal.1733750572795 2024-12-09T13:22:52,815 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34323:34323),(127.0.0.1/127.0.0.1:34135:34135),(127.0.0.1/127.0.0.1:44161:44161)] 2024-12-09T13:22:52,815 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7515d0d2e8096021274a3284a86db2a6, NAME => 'testReplayEditsWrittenViaHRegion,,1733750572636.7515d0d2e8096021274a3284a86db2a6.', STARTKEY => '', ENDKEY => ''} 2024-12-09T13:22:52,815 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733750572636.7515d0d2e8096021274a3284a86db2a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:52,815 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:52,815 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:52,817 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:52,818 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7515d0d2e8096021274a3284a86db2a6 columnFamilyName a 2024-12-09T13:22:52,818 DEBUG [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:52,819 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] regionserver.HStore(327): Store=7515d0d2e8096021274a3284a86db2a6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:52,819 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:52,820 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7515d0d2e8096021274a3284a86db2a6 columnFamilyName b 2024-12-09T13:22:52,820 DEBUG [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:52,820 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] regionserver.HStore(327): Store=7515d0d2e8096021274a3284a86db2a6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:52,821 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:52,821 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7515d0d2e8096021274a3284a86db2a6 columnFamilyName c 2024-12-09T13:22:52,821 DEBUG [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:52,822 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] regionserver.HStore(327): Store=7515d0d2e8096021274a3284a86db2a6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:52,822 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:52,823 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:52,825 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:52,825 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:52,826 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:52,826 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T13:22:52,827 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:52,828 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7515d0d2e8096021274a3284a86db2a6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66596478, jitterRate=-0.007635146379470825}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T13:22:52,829 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7515d0d2e8096021274a3284a86db2a6: Writing region info on filesystem at 1733750572815Initializing all the Stores at 1733750572816 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750572816Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750572817 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750572817Cleaning up temporary data from old regions at 1733750572826 (+9 ms)Region opened successfully at 1733750572829 (+3 ms) 2024-12-09T13:22:52,868 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7515d0d2e8096021274a3284a86db2a6 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-09T13:22:52,888 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/.tmp/a/3cd5e86595ce4588b6c8a08486b3d07d is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733750572829/Put/seqid=0 2024-12-09T13:22:52,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741895_1073 (size=5958) 2024-12-09T13:22:52,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741895_1073 (size=5958) 2024-12-09T13:22:52,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741895_1073 (size=5958) 2024-12-09T13:22:53,318 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/.tmp/a/3cd5e86595ce4588b6c8a08486b3d07d 2024-12-09T13:22:53,338 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/.tmp/b/45345c97e6b5489093e99fe8a593a1c5 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733750572839/Put/seqid=0 2024-12-09T13:22:53,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741896_1074 (size=5958) 2024-12-09T13:22:53,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741896_1074 (size=5958) 2024-12-09T13:22:53,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741896_1074 (size=5958) 2024-12-09T13:22:53,346 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/.tmp/b/45345c97e6b5489093e99fe8a593a1c5 2024-12-09T13:22:53,366 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/.tmp/c/247c1c221c784138b2b1ca0653acc627 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733750572850/Put/seqid=0 2024-12-09T13:22:53,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741897_1075 (size=5958) 2024-12-09T13:22:53,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741897_1075 (size=5958) 2024-12-09T13:22:53,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741897_1075 (size=5958) 2024-12-09T13:22:53,373 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/.tmp/c/247c1c221c784138b2b1ca0653acc627 2024-12-09T13:22:53,378 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/.tmp/a/3cd5e86595ce4588b6c8a08486b3d07d as hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/a/3cd5e86595ce4588b6c8a08486b3d07d 2024-12-09T13:22:53,382 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/a/3cd5e86595ce4588b6c8a08486b3d07d, entries=10, sequenceid=33, filesize=5.8 K 2024-12-09T13:22:53,383 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/.tmp/b/45345c97e6b5489093e99fe8a593a1c5 as hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/b/45345c97e6b5489093e99fe8a593a1c5 2024-12-09T13:22:53,388 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/b/45345c97e6b5489093e99fe8a593a1c5, entries=10, sequenceid=33, filesize=5.8 K 2024-12-09T13:22:53,389 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/.tmp/c/247c1c221c784138b2b1ca0653acc627 as hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/c/247c1c221c784138b2b1ca0653acc627 2024-12-09T13:22:53,393 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/c/247c1c221c784138b2b1ca0653acc627, entries=10, sequenceid=33, filesize=5.8 K 2024-12-09T13:22:53,394 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 7515d0d2e8096021274a3284a86db2a6 in 526ms, sequenceid=33, compaction requested=false 2024-12-09T13:22:53,394 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7515d0d2e8096021274a3284a86db2a6: 2024-12-09T13:22:53,394 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7515d0d2e8096021274a3284a86db2a6, disabling compactions & flushes 2024-12-09T13:22:53,394 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733750572636.7515d0d2e8096021274a3284a86db2a6. 2024-12-09T13:22:53,394 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733750572636.7515d0d2e8096021274a3284a86db2a6. 2024-12-09T13:22:53,394 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733750572636.7515d0d2e8096021274a3284a86db2a6. after waiting 0 ms 2024-12-09T13:22:53,394 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733750572636.7515d0d2e8096021274a3284a86db2a6. 2024-12-09T13:22:53,395 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733750572636.7515d0d2e8096021274a3284a86db2a6. 2024-12-09T13:22:53,395 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7515d0d2e8096021274a3284a86db2a6: Waiting for close lock at 1733750573394Disabling compacts and flushes for region at 1733750573394Disabling writes for close at 1733750573394Writing region close event to WAL at 1733750573395 (+1 ms)Closed at 1733750573395 2024-12-09T13:22:53,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741894_1072 (size=5937) 2024-12-09T13:22:53,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741894_1072 (size=5937) 2024-12-09T13:22:53,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741894_1072 (size=5937) 2024-12-09T13:22:53,402 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/b/45345c97e6b5489093e99fe8a593a1c5 to hdfs://localhost:41625/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/b/45345c97e6b5489093e99fe8a593a1c5 2024-12-09T13:22:53,418 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41625/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733750572635/wal.1733750572795, size=5.8 K (5937bytes) 2024-12-09T13:22:53,418 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41625/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733750572635/wal.1733750572795 2024-12-09T13:22:53,419 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41625/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733750572635/wal.1733750572795 after 1ms 2024-12-09T13:22:53,421 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41625/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733750572635/wal.1733750572795 took 3ms 2024-12-09T13:22:53,423 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41625/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733750572635/wal.1733750572795 so closing down 2024-12-09T13:22:53,423 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T13:22:53,424 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733750572795.temp 2024-12-09T13:22:53,425 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/recovered.edits/0000000000000000003-wal.1733750572795.temp 2024-12-09T13:22:53,425 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T13:22:53,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741898_1076 (size=5223) 2024-12-09T13:22:53,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741898_1076 (size=5223) 2024-12-09T13:22:53,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741898_1076 (size=5223) 2024-12-09T13:22:53,432 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/recovered.edits/0000000000000000003-wal.1733750572795.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-09T13:22:53,433 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/recovered.edits/0000000000000000003-wal.1733750572795.temp to hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/recovered.edits/0000000000000000032 2024-12-09T13:22:53,433 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:41625/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733750572635/wal.1733750572795, size=5.8 K, length=5937, corrupted=false, cancelled=false 2024-12-09T13:22:53,433 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41625/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733750572635/wal.1733750572795, journal: Splitting hdfs://localhost:41625/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733750572635/wal.1733750572795, size=5.8 K (5937bytes) at 1733750573418Finishing writing output for hdfs://localhost:41625/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733750572635/wal.1733750572795 so closing down at 1733750573423 (+5 ms)Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/recovered.edits/0000000000000000003-wal.1733750572795.temp at 1733750573425 (+2 ms)3 split writer threads finished at 1733750573425Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/recovered.edits/0000000000000000003-wal.1733750572795.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733750573432 (+7 ms)Rename recovered edits hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/recovered.edits/0000000000000000003-wal.1733750572795.temp to hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/recovered.edits/0000000000000000032 at 1733750573433 (+1 ms)Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:41625/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733750572635/wal.1733750572795, size=5.8 K, length=5937, corrupted=false, cancelled=false at 1733750573433 2024-12-09T13:22:53,435 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41625/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733750572635/wal.1733750572795 to hdfs://localhost:41625/hbase/oldWALs/wal.1733750572795 2024-12-09T13:22:53,436 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/recovered.edits/0000000000000000032 2024-12-09T13:22:53,436 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T13:22:53,438 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733750572635, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:22:53,451 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733750572635/wal.1733750573438, exclude list is [], retry=0 2024-12-09T13:22:53,454 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:22:53,454 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:22:53,455 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:22:53,456 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733750572635/wal.1733750573438 2024-12-09T13:22:53,457 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34135:34135),(127.0.0.1/127.0.0.1:34323:34323),(127.0.0.1/127.0.0.1:44161:44161)] 2024-12-09T13:22:53,457 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7515d0d2e8096021274a3284a86db2a6, NAME => 'testReplayEditsWrittenViaHRegion,,1733750572636.7515d0d2e8096021274a3284a86db2a6.', STARTKEY => '', ENDKEY => ''} 2024-12-09T13:22:53,457 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733750572636.7515d0d2e8096021274a3284a86db2a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:53,457 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:53,457 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:53,458 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:53,459 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7515d0d2e8096021274a3284a86db2a6 columnFamilyName a 2024-12-09T13:22:53,459 DEBUG [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:53,466 DEBUG [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/a/3cd5e86595ce4588b6c8a08486b3d07d 2024-12-09T13:22:53,466 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] regionserver.HStore(327): Store=7515d0d2e8096021274a3284a86db2a6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:53,466 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:53,467 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7515d0d2e8096021274a3284a86db2a6 columnFamilyName b 2024-12-09T13:22:53,467 DEBUG [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:53,468 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] regionserver.HStore(327): Store=7515d0d2e8096021274a3284a86db2a6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:53,468 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:53,468 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7515d0d2e8096021274a3284a86db2a6 columnFamilyName c 2024-12-09T13:22:53,468 DEBUG [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:53,474 DEBUG [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/c/247c1c221c784138b2b1ca0653acc627 2024-12-09T13:22:53,474 INFO [StoreOpener-7515d0d2e8096021274a3284a86db2a6-1 {}] regionserver.HStore(327): Store=7515d0d2e8096021274a3284a86db2a6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:53,474 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:53,475 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:53,476 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:53,477 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/recovered.edits/0000000000000000032 2024-12-09T13:22:53,480 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/recovered.edits/0000000000000000032 2024-12-09T13:22:53,480 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7515d0d2e8096021274a3284a86db2a6 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-09T13:22:53,494 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/.tmp/b/31bf6abe232d42ac8e8e3e61fd7de4fb is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733750572839/Put/seqid=0 2024-12-09T13:22:53,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741900_1078 (size=5958) 2024-12-09T13:22:53,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741900_1078 (size=5958) 2024-12-09T13:22:53,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741900_1078 (size=5958) 2024-12-09T13:22:53,502 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/.tmp/b/31bf6abe232d42ac8e8e3e61fd7de4fb 2024-12-09T13:22:53,509 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/.tmp/b/31bf6abe232d42ac8e8e3e61fd7de4fb as hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/b/31bf6abe232d42ac8e8e3e61fd7de4fb 2024-12-09T13:22:53,514 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/b/31bf6abe232d42ac8e8e3e61fd7de4fb, entries=10, sequenceid=32, filesize=5.8 K 2024-12-09T13:22:53,514 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 7515d0d2e8096021274a3284a86db2a6 in 34ms, sequenceid=32, compaction requested=false; wal=null 2024-12-09T13:22:53,515 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/recovered.edits/0000000000000000032 2024-12-09T13:22:53,516 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:53,516 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:53,517 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T13:22:53,518 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7515d0d2e8096021274a3284a86db2a6 2024-12-09T13:22:53,520 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/7515d0d2e8096021274a3284a86db2a6/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-09T13:22:53,521 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7515d0d2e8096021274a3284a86db2a6; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67935216, jitterRate=0.012313604354858398}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T13:22:53,521 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7515d0d2e8096021274a3284a86db2a6: Writing region info on filesystem at 1733750573457Initializing all the Stores at 1733750573458 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750573458Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750573458Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750573458Obtaining lock to block concurrent updates at 1733750573480 (+22 ms)Preparing flush snapshotting stores in 7515d0d2e8096021274a3284a86db2a6 at 1733750573480Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733750572636.7515d0d2e8096021274a3284a86db2a6., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733750573480Flushing stores of testReplayEditsWrittenViaHRegion,,1733750572636.7515d0d2e8096021274a3284a86db2a6. at 1733750573480Flushing 7515d0d2e8096021274a3284a86db2a6/b: creating writer at 1733750573480Flushing 7515d0d2e8096021274a3284a86db2a6/b: appending metadata at 1733750573494 (+14 ms)Flushing 7515d0d2e8096021274a3284a86db2a6/b: closing flushed file at 1733750573494Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d4c22fb: reopening flushed file at 1733750573508 (+14 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 7515d0d2e8096021274a3284a86db2a6 in 34ms, sequenceid=32, compaction requested=false; wal=null at 1733750573514 (+6 ms)Cleaning up temporary data from old regions at 1733750573516 (+2 ms)Region opened successfully at 1733750573521 (+5 ms) 2024-12-09T13:22:53,544 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=412 (was 403) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:35588 [Waiting for operation #31] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:43636 [Waiting for operation #44] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1859740653-172.17.0.3-1733750541580:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:43754 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:41496 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1859740653-172.17.0.3-1733750541580:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1859740653-172.17.0.3-1733750541580:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:41352 [Waiting for operation #27] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:35740 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1124 (was 1048) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=724 (was 724), ProcessCount=11 (was 11), AvailableMemoryMB=3564 (was 3573) 2024-12-09T13:22:53,544 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-12-09T13:22:53,558 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=412, OpenFileDescriptor=1124, MaxFileDescriptor=1048576, SystemLoadAverage=724, ProcessCount=11, AvailableMemoryMB=3563 2024-12-09T13:22:53,558 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-12-09T13:22:53,573 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:22:53,575 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:22:53,576 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T13:22:53,578 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-45268732, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/hregion-45268732, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:22:53,591 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-45268732/hregion-45268732.1733750573578, exclude list is [], retry=0 2024-12-09T13:22:53,594 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:22:53,595 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:22:53,595 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:22:53,598 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-45268732/hregion-45268732.1733750573578 2024-12-09T13:22:53,598 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34135:34135),(127.0.0.1/127.0.0.1:34323:34323),(127.0.0.1/127.0.0.1:44161:44161)] 2024-12-09T13:22:53,599 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1f2642fd11fae5989a25934f85920899, NAME => 'testReplayEditsAfterAbortingFlush,,1733750573574.1f2642fd11fae5989a25934f85920899.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41625/hbase 2024-12-09T13:22:53,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741902_1080 (size=68) 2024-12-09T13:22:53,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741902_1080 (size=68) 2024-12-09T13:22:53,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741902_1080 (size=68) 2024-12-09T13:22:54,011 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733750573574.1f2642fd11fae5989a25934f85920899.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:54,013 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1f2642fd11fae5989a25934f85920899 2024-12-09T13:22:54,014 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f2642fd11fae5989a25934f85920899 columnFamilyName a 2024-12-09T13:22:54,015 DEBUG [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:54,015 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] regionserver.HStore(327): Store=1f2642fd11fae5989a25934f85920899/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:54,015 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1f2642fd11fae5989a25934f85920899 2024-12-09T13:22:54,016 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f2642fd11fae5989a25934f85920899 columnFamilyName b 2024-12-09T13:22:54,016 DEBUG [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:54,017 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] regionserver.HStore(327): Store=1f2642fd11fae5989a25934f85920899/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:54,017 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1f2642fd11fae5989a25934f85920899 2024-12-09T13:22:54,018 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f2642fd11fae5989a25934f85920899 columnFamilyName c 2024-12-09T13:22:54,018 DEBUG [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:54,018 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] regionserver.HStore(327): Store=1f2642fd11fae5989a25934f85920899/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:54,019 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1f2642fd11fae5989a25934f85920899 2024-12-09T13:22:54,019 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899 2024-12-09T13:22:54,020 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899 2024-12-09T13:22:54,020 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1f2642fd11fae5989a25934f85920899 2024-12-09T13:22:54,020 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1f2642fd11fae5989a25934f85920899 2024-12-09T13:22:54,021 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T13:22:54,022 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1f2642fd11fae5989a25934f85920899 2024-12-09T13:22:54,024 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T13:22:54,024 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1f2642fd11fae5989a25934f85920899; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66688817, jitterRate=-0.0062591880559921265}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T13:22:54,025 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1f2642fd11fae5989a25934f85920899: Writing region info on filesystem at 1733750574012Initializing all the Stores at 1733750574012Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750574013 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750574013Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750574013Cleaning up temporary data from old regions at 1733750574021 (+8 ms)Region opened successfully at 1733750574025 (+4 ms) 2024-12-09T13:22:54,025 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1f2642fd11fae5989a25934f85920899, disabling compactions & flushes 2024-12-09T13:22:54,025 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733750573574.1f2642fd11fae5989a25934f85920899. 2024-12-09T13:22:54,025 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733750573574.1f2642fd11fae5989a25934f85920899. 2024-12-09T13:22:54,025 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733750573574.1f2642fd11fae5989a25934f85920899. after waiting 0 ms 2024-12-09T13:22:54,025 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733750573574.1f2642fd11fae5989a25934f85920899. 2024-12-09T13:22:54,026 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733750573574.1f2642fd11fae5989a25934f85920899. 2024-12-09T13:22:54,026 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1f2642fd11fae5989a25934f85920899: Waiting for close lock at 1733750574025Disabling compacts and flushes for region at 1733750574025Disabling writes for close at 1733750574025Writing region close event to WAL at 1733750574026 (+1 ms)Closed at 1733750574026 2024-12-09T13:22:54,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741901_1079 (size=93) 2024-12-09T13:22:54,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741901_1079 (size=93) 2024-12-09T13:22:54,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741901_1079 (size=93) 2024-12-09T13:22:54,030 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T13:22:54,030 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-45268732:(num 1733750573578) 2024-12-09T13:22:54,030 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T13:22:54,032 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733750573573, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:22:54,047 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733750573573/wal.1733750574033, exclude list is [], retry=0 2024-12-09T13:22:54,049 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:22:54,050 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:22:54,050 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:22:54,055 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733750573573/wal.1733750574033 2024-12-09T13:22:54,055 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34323:34323),(127.0.0.1/127.0.0.1:34135:34135)] 2024-12-09T13:22:54,122 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1f2642fd11fae5989a25934f85920899, NAME => 'testReplayEditsAfterAbortingFlush,,1733750573574.1f2642fd11fae5989a25934f85920899.', STARTKEY => '', ENDKEY => ''} 2024-12-09T13:22:54,125 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 1f2642fd11fae5989a25934f85920899 2024-12-09T13:22:54,125 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733750573574.1f2642fd11fae5989a25934f85920899.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:22:54,125 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1f2642fd11fae5989a25934f85920899 2024-12-09T13:22:54,125 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1f2642fd11fae5989a25934f85920899 2024-12-09T13:22:54,127 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1f2642fd11fae5989a25934f85920899 2024-12-09T13:22:54,128 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f2642fd11fae5989a25934f85920899 columnFamilyName a 2024-12-09T13:22:54,128 DEBUG [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:54,129 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] regionserver.HStore(327): Store=1f2642fd11fae5989a25934f85920899/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:54,129 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1f2642fd11fae5989a25934f85920899 2024-12-09T13:22:54,130 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f2642fd11fae5989a25934f85920899 columnFamilyName b 2024-12-09T13:22:54,130 DEBUG [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:54,130 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] regionserver.HStore(327): Store=1f2642fd11fae5989a25934f85920899/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:54,131 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1f2642fd11fae5989a25934f85920899 2024-12-09T13:22:54,131 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f2642fd11fae5989a25934f85920899 columnFamilyName c 2024-12-09T13:22:54,131 DEBUG [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:22:54,132 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] regionserver.HStore(327): Store=1f2642fd11fae5989a25934f85920899/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:22:54,132 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1f2642fd11fae5989a25934f85920899 2024-12-09T13:22:54,133 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899 2024-12-09T13:22:54,134 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899 2024-12-09T13:22:54,135 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1f2642fd11fae5989a25934f85920899 2024-12-09T13:22:54,135 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1f2642fd11fae5989a25934f85920899 2024-12-09T13:22:54,135 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T13:22:54,137 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1f2642fd11fae5989a25934f85920899 2024-12-09T13:22:54,137 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1f2642fd11fae5989a25934f85920899; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64871242, jitterRate=-0.03334316611289978}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T13:22:54,137 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1f2642fd11fae5989a25934f85920899 2024-12-09T13:22:54,138 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1f2642fd11fae5989a25934f85920899: Running coprocessor pre-open hook at 1733750574125Writing region info on filesystem at 1733750574125Initializing all the Stores at 1733750574126 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750574127 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750574127Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750574127Cleaning up temporary data from old regions at 1733750574135 (+8 ms)Running coprocessor post-open hooks at 1733750574138 (+3 ms)Region opened successfully at 1733750574138 2024-12-09T13:22:54,151 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1f2642fd11fae5989a25934f85920899 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-09T13:22:54,152 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 1f2642fd11fae5989a25934f85920899/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:22:55,153 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 1f2642fd11fae5989a25934f85920899/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:22:55,164 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T13:22:55,674 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T13:22:56,153 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 1f2642fd11fae5989a25934f85920899/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:22:57,154 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 1f2642fd11fae5989a25934f85920899/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:22:57,499 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-09T13:22:57,499 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-09T13:22:57,500 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-09T13:22:57,500 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-09T13:22:57,500 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-09T13:22:57,500 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-09T13:22:58,155 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 1f2642fd11fae5989a25934f85920899/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:22:59,155 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 1f2642fd11fae5989a25934f85920899/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:23:00,156 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 1f2642fd11fae5989a25934f85920899/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:23:01,157 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 1f2642fd11fae5989a25934f85920899/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:23:02,158 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 1f2642fd11fae5989a25934f85920899/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:23:03,159 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 1f2642fd11fae5989a25934f85920899/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:23:03,160 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1f2642fd11fae5989a25934f85920899: 2024-12-09T13:23:03,161 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733750573574.1f2642fd11fae5989a25934f85920899. 2024-12-09T13:23:03,182 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1f2642fd11fae5989a25934f85920899: 2024-12-09T13:23:03,183 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-09T13:23:03,183 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1f2642fd11fae5989a25934f85920899, disabling compactions & flushes 2024-12-09T13:23:03,183 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733750573574.1f2642fd11fae5989a25934f85920899. 2024-12-09T13:23:03,183 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733750573574.1f2642fd11fae5989a25934f85920899. 2024-12-09T13:23:03,183 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733750573574.1f2642fd11fae5989a25934f85920899. after waiting 0 ms 2024-12-09T13:23:03,183 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733750573574.1f2642fd11fae5989a25934f85920899. 2024-12-09T13:23:03,186 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733750573574.1f2642fd11fae5989a25934f85920899. 2024-12-09T13:23:03,186 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733750573574.1f2642fd11fae5989a25934f85920899. 2024-12-09T13:23:03,186 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1f2642fd11fae5989a25934f85920899: Waiting for close lock at 1733750583183Running coprocessor pre-close hooks at 1733750583183Disabling compacts and flushes for region at 1733750583183Disabling writes for close at 1733750583183Writing region close event to WAL at 1733750583185 (+2 ms)Running coprocessor post-close hooks at 1733750583186 (+1 ms)Closed at 1733750583186 2024-12-09T13:23:03,207 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733750573573/wal.1733750574033 not finished, retry = 0 2024-12-09T13:23:03,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741903_1081 (size=3941) 2024-12-09T13:23:03,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741903_1081 (size=3941) 2024-12-09T13:23:03,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741903_1081 (size=3941) 2024-12-09T13:23:03,334 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41625/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733750573573/wal.1733750574033, size=3.8 K (3941bytes) 2024-12-09T13:23:03,334 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41625/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733750573573/wal.1733750574033 2024-12-09T13:23:03,334 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41625/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733750573573/wal.1733750574033 after 0ms 2024-12-09T13:23:03,337 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41625/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733750573573/wal.1733750574033 took 4ms 2024-12-09T13:23:03,344 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41625/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733750573573/wal.1733750574033 so closing down 2024-12-09T13:23:03,344 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T13:23:03,355 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733750574033.temp 2024-12-09T13:23:03,357 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/recovered.edits/0000000000000000004-wal.1733750574033.temp 2024-12-09T13:23:03,358 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T13:23:03,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741904_1082 (size=2983) 2024-12-09T13:23:03,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741904_1082 (size=2983) 2024-12-09T13:23:03,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741904_1082 (size=2983) 2024-12-09T13:23:03,426 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/recovered.edits/0000000000000000004-wal.1733750574033.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-09T13:23:03,446 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/recovered.edits/0000000000000000004-wal.1733750574033.temp to hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/recovered.edits/0000000000000000026 2024-12-09T13:23:03,447 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 107 ms; skipped=3; WAL=hdfs://localhost:41625/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733750573573/wal.1733750574033, size=3.8 K, length=3941, corrupted=false, cancelled=false 2024-12-09T13:23:03,447 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41625/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733750573573/wal.1733750574033, journal: Splitting hdfs://localhost:41625/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733750573573/wal.1733750574033, size=3.8 K (3941bytes) at 1733750583334Finishing writing output for hdfs://localhost:41625/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733750573573/wal.1733750574033 so closing down at 1733750583344 (+10 ms)Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/recovered.edits/0000000000000000004-wal.1733750574033.temp at 1733750583357 (+13 ms)3 split writer threads finished at 1733750583358 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/recovered.edits/0000000000000000004-wal.1733750574033.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733750583427 (+69 ms)Rename recovered edits hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/recovered.edits/0000000000000000004-wal.1733750574033.temp to hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/recovered.edits/0000000000000000026 at 1733750583446 (+19 ms)Processed 23 edits across 1 Regions in 107 ms; skipped=3; WAL=hdfs://localhost:41625/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733750573573/wal.1733750574033, size=3.8 K, length=3941, corrupted=false, cancelled=false at 1733750583447 (+1 ms) 2024-12-09T13:23:03,453 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41625/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733750573573/wal.1733750574033 to hdfs://localhost:41625/hbase/oldWALs/wal.1733750574033 2024-12-09T13:23:03,454 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/recovered.edits/0000000000000000026 2024-12-09T13:23:03,454 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T13:23:03,460 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733750573573, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:23:03,482 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733750573573/wal.1733750583461, exclude list is [], retry=0 2024-12-09T13:23:03,499 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:23:03,499 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:23:03,499 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:23:03,515 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733750573573/wal.1733750583461 2024-12-09T13:23:03,520 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34135:34135),(127.0.0.1/127.0.0.1:34323:34323)] 2024-12-09T13:23:03,522 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1f2642fd11fae5989a25934f85920899, NAME => 'testReplayEditsAfterAbortingFlush,,1733750573574.1f2642fd11fae5989a25934f85920899.', STARTKEY => '', ENDKEY => ''} 2024-12-09T13:23:03,523 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 1f2642fd11fae5989a25934f85920899 2024-12-09T13:23:03,523 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733750573574.1f2642fd11fae5989a25934f85920899.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:23:03,523 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1f2642fd11fae5989a25934f85920899 2024-12-09T13:23:03,523 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1f2642fd11fae5989a25934f85920899 2024-12-09T13:23:03,535 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1f2642fd11fae5989a25934f85920899 2024-12-09T13:23:03,542 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f2642fd11fae5989a25934f85920899 columnFamilyName a 2024-12-09T13:23:03,542 DEBUG [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:03,557 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] regionserver.HStore(327): Store=1f2642fd11fae5989a25934f85920899/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:03,557 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1f2642fd11fae5989a25934f85920899 2024-12-09T13:23:03,562 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f2642fd11fae5989a25934f85920899 columnFamilyName b 2024-12-09T13:23:03,562 DEBUG [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:03,563 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] regionserver.HStore(327): Store=1f2642fd11fae5989a25934f85920899/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:03,564 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1f2642fd11fae5989a25934f85920899 2024-12-09T13:23:03,565 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f2642fd11fae5989a25934f85920899 columnFamilyName c 2024-12-09T13:23:03,565 DEBUG [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:03,566 INFO [StoreOpener-1f2642fd11fae5989a25934f85920899-1 {}] regionserver.HStore(327): Store=1f2642fd11fae5989a25934f85920899/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:03,566 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1f2642fd11fae5989a25934f85920899 2024-12-09T13:23:03,589 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899 2024-12-09T13:23:03,600 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899 2024-12-09T13:23:03,601 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/recovered.edits/0000000000000000026 2024-12-09T13:23:03,606 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/recovered.edits/0000000000000000026 2024-12-09T13:23:03,607 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1f2642fd11fae5989a25934f85920899 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-09T13:23:03,646 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/.tmp/a/5dbe4906eebb4adbbcbb501ff4bd07e4 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733750583168/Put/seqid=0 2024-12-09T13:23:03,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741906_1084 (size=5523) 2024-12-09T13:23:03,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741906_1084 (size=5523) 2024-12-09T13:23:03,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741906_1084 (size=5523) 2024-12-09T13:23:04,102 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/.tmp/a/5dbe4906eebb4adbbcbb501ff4bd07e4 2024-12-09T13:23:04,143 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/.tmp/b/237f2a29e72a4a93a145c52b4e2eeaec is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733750583161/Put/seqid=0 2024-12-09T13:23:04,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741907_1085 (size=5524) 2024-12-09T13:23:04,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741907_1085 (size=5524) 2024-12-09T13:23:04,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741907_1085 (size=5524) 2024-12-09T13:23:04,162 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/.tmp/b/237f2a29e72a4a93a145c52b4e2eeaec 2024-12-09T13:23:04,192 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/.tmp/c/57c8a3b5449d4ddf8af735608d10a007 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733750583165/Put/seqid=0 2024-12-09T13:23:04,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741908_1086 (size=5457) 2024-12-09T13:23:04,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741908_1086 (size=5457) 2024-12-09T13:23:04,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741908_1086 (size=5457) 2024-12-09T13:23:04,213 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/.tmp/c/57c8a3b5449d4ddf8af735608d10a007 2024-12-09T13:23:04,226 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/.tmp/a/5dbe4906eebb4adbbcbb501ff4bd07e4 as hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/a/5dbe4906eebb4adbbcbb501ff4bd07e4 2024-12-09T13:23:04,243 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/a/5dbe4906eebb4adbbcbb501ff4bd07e4, entries=7, sequenceid=26, filesize=5.4 K 2024-12-09T13:23:04,250 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/.tmp/b/237f2a29e72a4a93a145c52b4e2eeaec as hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/b/237f2a29e72a4a93a145c52b4e2eeaec 2024-12-09T13:23:04,268 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/b/237f2a29e72a4a93a145c52b4e2eeaec, entries=7, sequenceid=26, filesize=5.4 K 2024-12-09T13:23:04,271 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/.tmp/c/57c8a3b5449d4ddf8af735608d10a007 as hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/c/57c8a3b5449d4ddf8af735608d10a007 2024-12-09T13:23:04,284 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/c/57c8a3b5449d4ddf8af735608d10a007, entries=6, sequenceid=26, filesize=5.3 K 2024-12-09T13:23:04,285 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 1f2642fd11fae5989a25934f85920899 in 678ms, sequenceid=26, compaction requested=false; wal=null 2024-12-09T13:23:04,286 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/recovered.edits/0000000000000000026 2024-12-09T13:23:04,298 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1f2642fd11fae5989a25934f85920899 2024-12-09T13:23:04,298 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1f2642fd11fae5989a25934f85920899 2024-12-09T13:23:04,302 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T13:23:04,304 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1f2642fd11fae5989a25934f85920899 2024-12-09T13:23:04,307 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/hbase/data/default/testReplayEditsAfterAbortingFlush/1f2642fd11fae5989a25934f85920899/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-09T13:23:04,308 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1f2642fd11fae5989a25934f85920899; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60293662, jitterRate=-0.10155442357063293}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T13:23:04,309 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1f2642fd11fae5989a25934f85920899 2024-12-09T13:23:04,310 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1f2642fd11fae5989a25934f85920899: Running coprocessor pre-open hook at 1733750583523Writing region info on filesystem at 1733750583523Initializing all the Stores at 1733750583531 (+8 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750583531Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750583534 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750583534Obtaining lock to block concurrent updates at 1733750583607 (+73 ms)Preparing flush snapshotting stores in 1f2642fd11fae5989a25934f85920899 at 1733750583607Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733750573574.1f2642fd11fae5989a25934f85920899., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733750583607Flushing stores of testReplayEditsAfterAbortingFlush,,1733750573574.1f2642fd11fae5989a25934f85920899. at 1733750583607Flushing 1f2642fd11fae5989a25934f85920899/a: creating writer at 1733750583607Flushing 1f2642fd11fae5989a25934f85920899/a: appending metadata at 1733750583646 (+39 ms)Flushing 1f2642fd11fae5989a25934f85920899/a: closing flushed file at 1733750583646Flushing 1f2642fd11fae5989a25934f85920899/b: creating writer at 1733750584110 (+464 ms)Flushing 1f2642fd11fae5989a25934f85920899/b: appending metadata at 1733750584142 (+32 ms)Flushing 1f2642fd11fae5989a25934f85920899/b: closing flushed file at 1733750584142Flushing 1f2642fd11fae5989a25934f85920899/c: creating writer at 1733750584169 (+27 ms)Flushing 1f2642fd11fae5989a25934f85920899/c: appending metadata at 1733750584192 (+23 ms)Flushing 1f2642fd11fae5989a25934f85920899/c: closing flushed file at 1733750584192Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3773280a: reopening flushed file at 1733750584225 (+33 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e0dcb95: reopening flushed file at 1733750584243 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63ff9b85: reopening flushed file at 1733750584269 (+26 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 1f2642fd11fae5989a25934f85920899 in 678ms, sequenceid=26, compaction requested=false; wal=null at 1733750584285 (+16 ms)Cleaning up temporary data from old regions at 1733750584298 (+13 ms)Running coprocessor post-open hooks at 1733750584309 (+11 ms)Region opened successfully at 1733750584310 (+1 ms) 2024-12-09T13:23:04,349 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=413 (was 412) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:49112 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:49954 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1859740653-172.17.0.3-1733750541580:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1859740653-172.17.0.3-1733750541580:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:58648 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:58660 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:49130 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:49968 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1859740653-172.17.0.3-1733750541580:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1186 (was 1124) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=684 (was 724), ProcessCount=11 (was 11), AvailableMemoryMB=3417 (was 3563) 2024-12-09T13:23:04,350 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1186 is superior to 1024 2024-12-09T13:23:04,367 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=413, OpenFileDescriptor=1186, MaxFileDescriptor=1048576, SystemLoadAverage=684, ProcessCount=11, AvailableMemoryMB=3415 2024-12-09T13:23:04,367 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1186 is superior to 1024 2024-12-09T13:23:04,386 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:23:04,389 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:23:04,390 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T13:23:04,393 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-89397907, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/hregion-89397907, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:23:04,410 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-89397907/hregion-89397907.1733750584394, exclude list is [], retry=0 2024-12-09T13:23:04,414 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:23:04,423 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:23:04,423 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:23:04,436 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-89397907/hregion-89397907.1733750584394 2024-12-09T13:23:04,436 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34135:34135),(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34323:34323)] 2024-12-09T13:23:04,436 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => df4c8dc1ff3a62026438ea6f21329699, NAME => 'testDatalossWhenInputError,,1733750584387.df4c8dc1ff3a62026438ea6f21329699.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41625/hbase 2024-12-09T13:23:04,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741910_1088 (size=61) 2024-12-09T13:23:04,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741910_1088 (size=61) 2024-12-09T13:23:04,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741910_1088 (size=61) 2024-12-09T13:23:04,467 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733750584387.df4c8dc1ff3a62026438ea6f21329699.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:23:04,470 INFO [StoreOpener-df4c8dc1ff3a62026438ea6f21329699-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:04,472 INFO [StoreOpener-df4c8dc1ff3a62026438ea6f21329699-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df4c8dc1ff3a62026438ea6f21329699 columnFamilyName a 2024-12-09T13:23:04,472 DEBUG [StoreOpener-df4c8dc1ff3a62026438ea6f21329699-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:04,472 INFO [StoreOpener-df4c8dc1ff3a62026438ea6f21329699-1 {}] regionserver.HStore(327): Store=df4c8dc1ff3a62026438ea6f21329699/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:04,472 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:04,473 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:04,474 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:04,474 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:04,474 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:04,482 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:04,491 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T13:23:04,491 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened df4c8dc1ff3a62026438ea6f21329699; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63456322, jitterRate=-0.054427117109298706}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T13:23:04,493 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for df4c8dc1ff3a62026438ea6f21329699: Writing region info on filesystem at 1733750584467Initializing all the Stores at 1733750584469 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750584470 (+1 ms)Cleaning up temporary data from old regions at 1733750584474 (+4 ms)Region opened successfully at 1733750584493 (+19 ms) 2024-12-09T13:23:04,493 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing df4c8dc1ff3a62026438ea6f21329699, disabling compactions & flushes 2024-12-09T13:23:04,493 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733750584387.df4c8dc1ff3a62026438ea6f21329699. 2024-12-09T13:23:04,493 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733750584387.df4c8dc1ff3a62026438ea6f21329699. 2024-12-09T13:23:04,493 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733750584387.df4c8dc1ff3a62026438ea6f21329699. after waiting 0 ms 2024-12-09T13:23:04,493 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733750584387.df4c8dc1ff3a62026438ea6f21329699. 2024-12-09T13:23:04,494 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733750584387.df4c8dc1ff3a62026438ea6f21329699. 2024-12-09T13:23:04,494 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for df4c8dc1ff3a62026438ea6f21329699: Waiting for close lock at 1733750584493Disabling compacts and flushes for region at 1733750584493Disabling writes for close at 1733750584493Writing region close event to WAL at 1733750584494 (+1 ms)Closed at 1733750584494 2024-12-09T13:23:04,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741909_1087 (size=93) 2024-12-09T13:23:04,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741909_1087 (size=93) 2024-12-09T13:23:04,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741909_1087 (size=93) 2024-12-09T13:23:04,512 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T13:23:04,512 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-89397907:(num 1733750584394) 2024-12-09T13:23:04,513 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T13:23:04,519 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/testdatalosswheninputerror-manual,16010,1733750584385, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:23:04,535 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733750584385/wal.1733750584520, exclude list is [], retry=0 2024-12-09T13:23:04,538 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:23:04,538 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:23:04,539 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:23:04,546 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733750584385/wal.1733750584520 2024-12-09T13:23:04,547 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34323:34323),(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34135:34135)] 2024-12-09T13:23:04,547 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => df4c8dc1ff3a62026438ea6f21329699, NAME => 'testDatalossWhenInputError,,1733750584387.df4c8dc1ff3a62026438ea6f21329699.', STARTKEY => '', ENDKEY => ''} 2024-12-09T13:23:04,547 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733750584387.df4c8dc1ff3a62026438ea6f21329699.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:23:04,547 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:04,547 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:04,549 INFO [StoreOpener-df4c8dc1ff3a62026438ea6f21329699-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:04,550 INFO [StoreOpener-df4c8dc1ff3a62026438ea6f21329699-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df4c8dc1ff3a62026438ea6f21329699 columnFamilyName a 2024-12-09T13:23:04,550 DEBUG [StoreOpener-df4c8dc1ff3a62026438ea6f21329699-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:04,551 INFO [StoreOpener-df4c8dc1ff3a62026438ea6f21329699-1 {}] regionserver.HStore(327): Store=df4c8dc1ff3a62026438ea6f21329699/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:04,551 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:04,552 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:04,553 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:04,553 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:04,553 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:04,555 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:04,556 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened df4c8dc1ff3a62026438ea6f21329699; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61798170, jitterRate=-0.07913550734519958}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T13:23:04,557 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for df4c8dc1ff3a62026438ea6f21329699: Writing region info on filesystem at 1733750584548Initializing all the Stores at 1733750584549 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750584549Cleaning up temporary data from old regions at 1733750584553 (+4 ms)Region opened successfully at 1733750584557 (+4 ms) 2024-12-09T13:23:04,595 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing df4c8dc1ff3a62026438ea6f21329699, disabling compactions & flushes 2024-12-09T13:23:04,596 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733750584387.df4c8dc1ff3a62026438ea6f21329699. 2024-12-09T13:23:04,596 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733750584387.df4c8dc1ff3a62026438ea6f21329699. 2024-12-09T13:23:04,596 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733750584387.df4c8dc1ff3a62026438ea6f21329699. after waiting 0 ms 2024-12-09T13:23:04,596 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733750584387.df4c8dc1ff3a62026438ea6f21329699. 2024-12-09T13:23:04,597 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733750584387.df4c8dc1ff3a62026438ea6f21329699. 2024-12-09T13:23:04,597 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733750584387.df4c8dc1ff3a62026438ea6f21329699. 2024-12-09T13:23:04,597 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for df4c8dc1ff3a62026438ea6f21329699: Waiting for close lock at 1733750584595Disabling compacts and flushes for region at 1733750584595Disabling writes for close at 1733750584596 (+1 ms)Writing region close event to WAL at 1733750584597 (+1 ms)Closed at 1733750584597 2024-12-09T13:23:04,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741911_1089 (size=1623) 2024-12-09T13:23:04,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741911_1089 (size=1623) 2024-12-09T13:23:04,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741911_1089 (size=1623) 2024-12-09T13:23:04,651 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41625/hbase/WALs/testdatalosswheninputerror-manual,16010,1733750584385/wal.1733750584520, size=1.6 K (1623bytes) 2024-12-09T13:23:04,652 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41625/hbase/WALs/testdatalosswheninputerror-manual,16010,1733750584385/wal.1733750584520 2024-12-09T13:23:04,652 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41625/hbase/WALs/testdatalosswheninputerror-manual,16010,1733750584385/wal.1733750584520 after 0ms 2024-12-09T13:23:04,654 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41625/hbase/WALs/testdatalosswheninputerror-manual,16010,1733750584385/wal.1733750584520 took 3ms 2024-12-09T13:23:04,668 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41625/hbase/WALs/testdatalosswheninputerror-manual,16010,1733750584385/wal.1733750584520 so closing down 2024-12-09T13:23:04,668 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T13:23:04,674 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733750584520.temp 2024-12-09T13:23:04,675 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699/recovered.edits/0000000000000000003-wal.1733750584520.temp 2024-12-09T13:23:04,676 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T13:23:04,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741912_1090 (size=1623) 2024-12-09T13:23:04,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741912_1090 (size=1623) 2024-12-09T13:23:04,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741912_1090 (size=1623) 2024-12-09T13:23:04,711 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699/recovered.edits/0000000000000000003-wal.1733750584520.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-09T13:23:04,713 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699/recovered.edits/0000000000000000003-wal.1733750584520.temp to hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699/recovered.edits/0000000000000000012 2024-12-09T13:23:04,713 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 51 ms; skipped=0; WAL=hdfs://localhost:41625/hbase/WALs/testdatalosswheninputerror-manual,16010,1733750584385/wal.1733750584520, size=1.6 K, length=1623, corrupted=false, cancelled=false 2024-12-09T13:23:04,713 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41625/hbase/WALs/testdatalosswheninputerror-manual,16010,1733750584385/wal.1733750584520, journal: Splitting hdfs://localhost:41625/hbase/WALs/testdatalosswheninputerror-manual,16010,1733750584385/wal.1733750584520, size=1.6 K (1623bytes) at 1733750584651Finishing writing output for hdfs://localhost:41625/hbase/WALs/testdatalosswheninputerror-manual,16010,1733750584385/wal.1733750584520 so closing down at 1733750584668 (+17 ms)Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699/recovered.edits/0000000000000000003-wal.1733750584520.temp at 1733750584675 (+7 ms)3 split writer threads finished at 1733750584676 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699/recovered.edits/0000000000000000003-wal.1733750584520.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733750584711 (+35 ms)Rename recovered edits hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699/recovered.edits/0000000000000000003-wal.1733750584520.temp to hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699/recovered.edits/0000000000000000012 at 1733750584713 (+2 ms)Processed 10 edits across 1 Regions in 51 ms; skipped=0; WAL=hdfs://localhost:41625/hbase/WALs/testdatalosswheninputerror-manual,16010,1733750584385/wal.1733750584520, size=1.6 K, length=1623, corrupted=false, cancelled=false at 1733750584713 2024-12-09T13:23:04,716 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41625/hbase/WALs/testdatalosswheninputerror-manual,16010,1733750584385/wal.1733750584520 to hdfs://localhost:41625/hbase/oldWALs/wal.1733750584520 2024-12-09T13:23:04,717 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699/recovered.edits/0000000000000000012 2024-12-09T13:23:05,259 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T13:23:05,264 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/testdatalosswheninputerror-manual,16010,1733750584385, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:23:05,286 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733750584385/wal.1733750585265, exclude list is [], retry=0 2024-12-09T13:23:05,291 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:23:05,291 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:23:05,291 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:23:05,298 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733750584385/wal.1733750585265 2024-12-09T13:23:05,302 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34323:34323),(127.0.0.1/127.0.0.1:34135:34135),(127.0.0.1/127.0.0.1:44161:44161)] 2024-12-09T13:23:05,302 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => df4c8dc1ff3a62026438ea6f21329699, NAME => 'testDatalossWhenInputError,,1733750584387.df4c8dc1ff3a62026438ea6f21329699.', STARTKEY => '', ENDKEY => ''} 2024-12-09T13:23:05,302 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733750584387.df4c8dc1ff3a62026438ea6f21329699.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:23:05,302 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:05,302 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:05,314 INFO [StoreOpener-df4c8dc1ff3a62026438ea6f21329699-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:05,316 INFO [StoreOpener-df4c8dc1ff3a62026438ea6f21329699-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df4c8dc1ff3a62026438ea6f21329699 columnFamilyName a 2024-12-09T13:23:05,316 DEBUG [StoreOpener-df4c8dc1ff3a62026438ea6f21329699-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:05,317 INFO [StoreOpener-df4c8dc1ff3a62026438ea6f21329699-1 {}] regionserver.HStore(327): Store=df4c8dc1ff3a62026438ea6f21329699/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:05,318 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:05,318 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:05,320 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:05,321 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699/recovered.edits/0000000000000000012 2024-12-09T13:23:05,330 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699/recovered.edits/0000000000000000012 2024-12-09T13:23:05,331 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing df4c8dc1ff3a62026438ea6f21329699 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-09T13:23:05,371 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699/.tmp/a/38bbf2263388437397b7a6389c9b2c64 is 79, key is testDatalossWhenInputError/a:x0/1733750584557/Put/seqid=0 2024-12-09T13:23:05,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741914_1092 (size=5808) 2024-12-09T13:23:05,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741914_1092 (size=5808) 2024-12-09T13:23:05,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741914_1092 (size=5808) 2024-12-09T13:23:05,395 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699/.tmp/a/38bbf2263388437397b7a6389c9b2c64 2024-12-09T13:23:05,411 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699/.tmp/a/38bbf2263388437397b7a6389c9b2c64 as hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699/a/38bbf2263388437397b7a6389c9b2c64 2024-12-09T13:23:05,428 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699/a/38bbf2263388437397b7a6389c9b2c64, entries=10, sequenceid=12, filesize=5.7 K 2024-12-09T13:23:05,428 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for df4c8dc1ff3a62026438ea6f21329699 in 97ms, sequenceid=12, compaction requested=false; wal=null 2024-12-09T13:23:05,429 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699/recovered.edits/0000000000000000012 2024-12-09T13:23:05,430 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:05,430 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:05,434 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:05,437 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-09T13:23:05,438 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened df4c8dc1ff3a62026438ea6f21329699; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61186577, jitterRate=-0.08824895322322845}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T13:23:05,440 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for df4c8dc1ff3a62026438ea6f21329699: Writing region info on filesystem at 1733750585302Initializing all the Stores at 1733750585313 (+11 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750585313Obtaining lock to block concurrent updates at 1733750585331 (+18 ms)Preparing flush snapshotting stores in df4c8dc1ff3a62026438ea6f21329699 at 1733750585331Finished memstore snapshotting testDatalossWhenInputError,,1733750584387.df4c8dc1ff3a62026438ea6f21329699., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733750585331Flushing stores of testDatalossWhenInputError,,1733750584387.df4c8dc1ff3a62026438ea6f21329699. at 1733750585331Flushing df4c8dc1ff3a62026438ea6f21329699/a: creating writer at 1733750585331Flushing df4c8dc1ff3a62026438ea6f21329699/a: appending metadata at 1733750585370 (+39 ms)Flushing df4c8dc1ff3a62026438ea6f21329699/a: closing flushed file at 1733750585370Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7cef4a59: reopening flushed file at 1733750585406 (+36 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for df4c8dc1ff3a62026438ea6f21329699 in 97ms, sequenceid=12, compaction requested=false; wal=null at 1733750585428 (+22 ms)Cleaning up temporary data from old regions at 1733750585430 (+2 ms)Region opened successfully at 1733750585440 (+10 ms) 2024-12-09T13:23:05,444 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => df4c8dc1ff3a62026438ea6f21329699, NAME => 'testDatalossWhenInputError,,1733750584387.df4c8dc1ff3a62026438ea6f21329699.', STARTKEY => '', ENDKEY => ''} 2024-12-09T13:23:05,444 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733750584387.df4c8dc1ff3a62026438ea6f21329699.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:23:05,444 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:05,444 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:05,471 INFO [StoreOpener-df4c8dc1ff3a62026438ea6f21329699-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:05,477 INFO [StoreOpener-df4c8dc1ff3a62026438ea6f21329699-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df4c8dc1ff3a62026438ea6f21329699 columnFamilyName a 2024-12-09T13:23:05,477 DEBUG [StoreOpener-df4c8dc1ff3a62026438ea6f21329699-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:05,502 DEBUG [StoreOpener-df4c8dc1ff3a62026438ea6f21329699-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699/a/38bbf2263388437397b7a6389c9b2c64 2024-12-09T13:23:05,502 INFO [StoreOpener-df4c8dc1ff3a62026438ea6f21329699-1 {}] regionserver.HStore(327): Store=df4c8dc1ff3a62026438ea6f21329699/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:05,503 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:05,504 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:05,508 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:05,508 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:05,509 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:05,510 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for df4c8dc1ff3a62026438ea6f21329699 2024-12-09T13:23:05,513 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/hbase/data/default/testDatalossWhenInputError/df4c8dc1ff3a62026438ea6f21329699/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-09T13:23:05,515 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened df4c8dc1ff3a62026438ea6f21329699; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58745657, jitterRate=-0.12462149560451508}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T13:23:05,515 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for df4c8dc1ff3a62026438ea6f21329699: Writing region info on filesystem at 1733750585444Initializing all the Stores at 1733750585452 (+8 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750585452Cleaning up temporary data from old regions at 1733750585509 (+57 ms)Region opened successfully at 1733750585515 (+6 ms) 2024-12-09T13:23:05,547 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=423 (was 413) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:49954 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1859740653-172.17.0.3-1733750541580:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1859740653-172.17.0.3-1733750541580:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:49130 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:49204 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:50044 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:58758 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1859740653-172.17.0.3-1733750541580:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1268 (was 1186) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=684 (was 684), ProcessCount=11 (was 11), AvailableMemoryMB=3335 (was 3415) 2024-12-09T13:23:05,548 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1268 is superior to 1024 2024-12-09T13:23:05,563 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=423, OpenFileDescriptor=1268, MaxFileDescriptor=1048576, SystemLoadAverage=684, ProcessCount=11, AvailableMemoryMB=3332 2024-12-09T13:23:05,563 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1268 is superior to 1024 2024-12-09T13:23:05,593 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:23:05,596 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:23:05,597 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T13:23:05,607 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-39809179, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/hregion-39809179, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:23:05,627 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-39809179/hregion-39809179.1733750585608, exclude list is [], retry=0 2024-12-09T13:23:05,637 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:23:05,639 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:23:05,643 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:23:05,653 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-39809179/hregion-39809179.1733750585608 2024-12-09T13:23:05,654 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34135:34135),(127.0.0.1/127.0.0.1:34323:34323)] 2024-12-09T13:23:05,654 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => eeff0a00bbc1a0fec494b91bb4c7c821, NAME => 'testCompactedBulkLoadedFiles,,1733750585594.eeff0a00bbc1a0fec494b91bb4c7c821.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41625/hbase 2024-12-09T13:23:05,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741916_1094 (size=63) 2024-12-09T13:23:05,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741916_1094 (size=63) 2024-12-09T13:23:05,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741916_1094 (size=63) 2024-12-09T13:23:05,686 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733750585594.eeff0a00bbc1a0fec494b91bb4c7c821.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:23:05,732 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:05,734 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eeff0a00bbc1a0fec494b91bb4c7c821 columnFamilyName a 2024-12-09T13:23:05,734 DEBUG [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:05,735 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.HStore(327): Store=eeff0a00bbc1a0fec494b91bb4c7c821/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:05,735 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:05,736 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eeff0a00bbc1a0fec494b91bb4c7c821 columnFamilyName b 2024-12-09T13:23:05,737 DEBUG [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:05,739 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.HStore(327): Store=eeff0a00bbc1a0fec494b91bb4c7c821/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:05,742 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:05,745 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eeff0a00bbc1a0fec494b91bb4c7c821 columnFamilyName c 2024-12-09T13:23:05,745 DEBUG [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:05,747 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.HStore(327): Store=eeff0a00bbc1a0fec494b91bb4c7c821/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:05,747 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:05,751 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:05,751 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:05,754 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:05,754 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:05,755 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T13:23:05,758 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:05,767 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T13:23:05,768 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened eeff0a00bbc1a0fec494b91bb4c7c821; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75168128, jitterRate=0.12009239196777344}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T13:23:05,769 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for eeff0a00bbc1a0fec494b91bb4c7c821: Writing region info on filesystem at 1733750585729Initializing all the Stores at 1733750585731 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750585731Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750585731Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750585732 (+1 ms)Cleaning up temporary data from old regions at 1733750585754 (+22 ms)Region opened successfully at 1733750585769 (+15 ms) 2024-12-09T13:23:05,769 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing eeff0a00bbc1a0fec494b91bb4c7c821, disabling compactions & flushes 2024-12-09T13:23:05,769 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733750585594.eeff0a00bbc1a0fec494b91bb4c7c821. 2024-12-09T13:23:05,769 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733750585594.eeff0a00bbc1a0fec494b91bb4c7c821. 2024-12-09T13:23:05,769 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733750585594.eeff0a00bbc1a0fec494b91bb4c7c821. after waiting 0 ms 2024-12-09T13:23:05,769 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733750585594.eeff0a00bbc1a0fec494b91bb4c7c821. 2024-12-09T13:23:05,770 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733750585594.eeff0a00bbc1a0fec494b91bb4c7c821. 2024-12-09T13:23:05,771 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for eeff0a00bbc1a0fec494b91bb4c7c821: Waiting for close lock at 1733750585769Disabling compacts and flushes for region at 1733750585769Disabling writes for close at 1733750585769Writing region close event to WAL at 1733750585770 (+1 ms)Closed at 1733750585770 2024-12-09T13:23:05,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741915_1093 (size=93) 2024-12-09T13:23:05,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741915_1093 (size=93) 2024-12-09T13:23:05,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741915_1093 (size=93) 2024-12-09T13:23:05,785 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T13:23:05,785 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-39809179:(num 1733750585608) 2024-12-09T13:23:05,785 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T13:23:05,788 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733750585592, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:23:05,803 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733750585592/wal.1733750585788, exclude list is [], retry=0 2024-12-09T13:23:05,805 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:23:05,806 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:23:05,806 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:23:05,808 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733750585592/wal.1733750585788 2024-12-09T13:23:05,809 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34323:34323),(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34135:34135)] 2024-12-09T13:23:05,809 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => eeff0a00bbc1a0fec494b91bb4c7c821, NAME => 'testCompactedBulkLoadedFiles,,1733750585594.eeff0a00bbc1a0fec494b91bb4c7c821.', STARTKEY => '', ENDKEY => ''} 2024-12-09T13:23:05,810 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733750585594.eeff0a00bbc1a0fec494b91bb4c7c821.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:23:05,810 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:05,810 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:05,813 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:05,813 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eeff0a00bbc1a0fec494b91bb4c7c821 columnFamilyName a 2024-12-09T13:23:05,813 DEBUG [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:05,814 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.HStore(327): Store=eeff0a00bbc1a0fec494b91bb4c7c821/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:05,814 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:05,815 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eeff0a00bbc1a0fec494b91bb4c7c821 columnFamilyName b 2024-12-09T13:23:05,815 DEBUG [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:05,816 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.HStore(327): Store=eeff0a00bbc1a0fec494b91bb4c7c821/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:05,816 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:05,816 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eeff0a00bbc1a0fec494b91bb4c7c821 columnFamilyName c 2024-12-09T13:23:05,816 DEBUG [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:05,817 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.HStore(327): Store=eeff0a00bbc1a0fec494b91bb4c7c821/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:05,817 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:05,818 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:05,819 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:05,820 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:05,820 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:05,821 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T13:23:05,828 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:05,829 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened eeff0a00bbc1a0fec494b91bb4c7c821; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59814190, jitterRate=-0.10869911313056946}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T13:23:05,830 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for eeff0a00bbc1a0fec494b91bb4c7c821: Writing region info on filesystem at 1733750585810Initializing all the Stores at 1733750585811 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750585811Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750585812 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750585812Cleaning up temporary data from old regions at 1733750585820 (+8 ms)Region opened successfully at 1733750585830 (+10 ms) 2024-12-09T13:23:05,837 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733750585837/Put/seqid=0 2024-12-09T13:23:05,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741918_1096 (size=4875) 2024-12-09T13:23:05,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741918_1096 (size=4875) 2024-12-09T13:23:05,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741918_1096 (size=4875) 2024-12-09T13:23:05,869 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733750585869/Put/seqid=0 2024-12-09T13:23:05,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741919_1097 (size=4875) 2024-12-09T13:23:05,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741919_1097 (size=4875) 2024-12-09T13:23:05,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741919_1097 (size=4875) 2024-12-09T13:23:05,895 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733750585895/Put/seqid=0 2024-12-09T13:23:05,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741920_1098 (size=4875) 2024-12-09T13:23:05,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741920_1098 (size=4875) 2024-12-09T13:23:05,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741920_1098 (size=4875) 2024-12-09T13:23:05,911 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41625/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in eeff0a00bbc1a0fec494b91bb4c7c821/a 2024-12-09T13:23:05,915 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-09T13:23:05,915 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T13:23:05,915 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41625/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in eeff0a00bbc1a0fec494b91bb4c7c821/a 2024-12-09T13:23:05,924 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-09T13:23:05,924 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T13:23:05,924 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41625/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in eeff0a00bbc1a0fec494b91bb4c7c821/a 2024-12-09T13:23:05,933 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-09T13:23:05,933 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T13:23:05,933 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing eeff0a00bbc1a0fec494b91bb4c7c821 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-09T13:23:05,957 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/.tmp/a/e6210a73856a4a9893f973549e1d7fc8 is 55, key is testCompactedBulkLoadedFiles/a:a/1733750585830/Put/seqid=0 2024-12-09T13:23:05,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741921_1099 (size=5107) 2024-12-09T13:23:05,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741921_1099 (size=5107) 2024-12-09T13:23:05,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741921_1099 (size=5107) 2024-12-09T13:23:06,375 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/.tmp/a/e6210a73856a4a9893f973549e1d7fc8 2024-12-09T13:23:06,381 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/.tmp/a/e6210a73856a4a9893f973549e1d7fc8 as hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/e6210a73856a4a9893f973549e1d7fc8 2024-12-09T13:23:06,387 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/e6210a73856a4a9893f973549e1d7fc8, entries=1, sequenceid=4, filesize=5.0 K 2024-12-09T13:23:06,389 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for eeff0a00bbc1a0fec494b91bb4c7c821 in 456ms, sequenceid=4, compaction requested=false 2024-12-09T13:23:06,389 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for eeff0a00bbc1a0fec494b91bb4c7c821: 2024-12-09T13:23:06,390 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/3e2231843afc4eb6aacfc2f3d020e739_SeqId_4_ 2024-12-09T13:23:06,392 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/3c03515cdb144048b6c45858c9671095_SeqId_4_ 2024-12-09T13:23:06,393 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/b061dccd38cc452cbe407fe0fd346be9_SeqId_4_ 2024-12-09T13:23:06,394 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41625/hbase/testCompactedBulkLoadedFiles/hfile0 into eeff0a00bbc1a0fec494b91bb4c7c821/a as hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/3e2231843afc4eb6aacfc2f3d020e739_SeqId_4_ - updating store file list. 2024-12-09T13:23:06,399 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3e2231843afc4eb6aacfc2f3d020e739_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T13:23:06,399 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/3e2231843afc4eb6aacfc2f3d020e739_SeqId_4_ into eeff0a00bbc1a0fec494b91bb4c7c821/a 2024-12-09T13:23:06,399 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41625/hbase/testCompactedBulkLoadedFiles/hfile0 into eeff0a00bbc1a0fec494b91bb4c7c821/a (new location: hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/3e2231843afc4eb6aacfc2f3d020e739_SeqId_4_) 2024-12-09T13:23:06,401 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41625/hbase/testCompactedBulkLoadedFiles/hfile1 into eeff0a00bbc1a0fec494b91bb4c7c821/a as hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/3c03515cdb144048b6c45858c9671095_SeqId_4_ - updating store file list. 2024-12-09T13:23:06,405 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3c03515cdb144048b6c45858c9671095_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T13:23:06,406 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/3c03515cdb144048b6c45858c9671095_SeqId_4_ into eeff0a00bbc1a0fec494b91bb4c7c821/a 2024-12-09T13:23:06,406 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41625/hbase/testCompactedBulkLoadedFiles/hfile1 into eeff0a00bbc1a0fec494b91bb4c7c821/a (new location: hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/3c03515cdb144048b6c45858c9671095_SeqId_4_) 2024-12-09T13:23:06,409 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41625/hbase/testCompactedBulkLoadedFiles/hfile2 into eeff0a00bbc1a0fec494b91bb4c7c821/a as hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/b061dccd38cc452cbe407fe0fd346be9_SeqId_4_ - updating store file list. 2024-12-09T13:23:06,414 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for b061dccd38cc452cbe407fe0fd346be9_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T13:23:06,414 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/b061dccd38cc452cbe407fe0fd346be9_SeqId_4_ into eeff0a00bbc1a0fec494b91bb4c7c821/a 2024-12-09T13:23:06,414 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41625/hbase/testCompactedBulkLoadedFiles/hfile2 into eeff0a00bbc1a0fec494b91bb4c7c821/a (new location: hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/b061dccd38cc452cbe407fe0fd346be9_SeqId_4_) 2024-12-09T13:23:06,423 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T13:23:06,423 DEBUG [Time-limited test {}] regionserver.HStore(1541): eeff0a00bbc1a0fec494b91bb4c7c821/a is initiating major compaction (all files) 2024-12-09T13:23:06,423 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of eeff0a00bbc1a0fec494b91bb4c7c821/a in testCompactedBulkLoadedFiles,,1733750585594.eeff0a00bbc1a0fec494b91bb4c7c821. 2024-12-09T13:23:06,423 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/e6210a73856a4a9893f973549e1d7fc8, hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/3e2231843afc4eb6aacfc2f3d020e739_SeqId_4_, hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/3c03515cdb144048b6c45858c9671095_SeqId_4_, hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/b061dccd38cc452cbe407fe0fd346be9_SeqId_4_] into tmpdir=hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/.tmp, totalSize=19.3 K 2024-12-09T13:23:06,424 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e6210a73856a4a9893f973549e1d7fc8, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733750585830 2024-12-09T13:23:06,424 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3e2231843afc4eb6aacfc2f3d020e739_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-09T13:23:06,425 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3c03515cdb144048b6c45858c9671095_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-09T13:23:06,425 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting b061dccd38cc452cbe407fe0fd346be9_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-09T13:23:06,449 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/.tmp/a/1bd52b11886f47ed968a0dad0629c63a is 55, key is testCompactedBulkLoadedFiles/a:a/1733750585830/Put/seqid=0 2024-12-09T13:23:06,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741922_1100 (size=6154) 2024-12-09T13:23:06,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741922_1100 (size=6154) 2024-12-09T13:23:06,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741922_1100 (size=6154) 2024-12-09T13:23:06,880 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/.tmp/a/1bd52b11886f47ed968a0dad0629c63a as hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/1bd52b11886f47ed968a0dad0629c63a 2024-12-09T13:23:06,889 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in eeff0a00bbc1a0fec494b91bb4c7c821/a of eeff0a00bbc1a0fec494b91bb4c7c821 into 1bd52b11886f47ed968a0dad0629c63a(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T13:23:06,889 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for eeff0a00bbc1a0fec494b91bb4c7c821: 2024-12-09T13:23:06,889 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-09T13:23:06,889 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-09T13:23:06,938 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41625/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733750585592/wal.1733750585788, size=0 (0bytes) 2024-12-09T13:23:06,938 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41625/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733750585592/wal.1733750585788 might be still open, length is 0 2024-12-09T13:23:06,938 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41625/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733750585592/wal.1733750585788 2024-12-09T13:23:06,939 WARN [IPC Server handler 0 on default port 41625 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733750585592/wal.1733750585788 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-12-09T13:23:06,939 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41625/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733750585592/wal.1733750585788 after 1ms 2024-12-09T13:23:07,274 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:49240 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:40963:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49240 dst: /127.0.0.1:40963 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:40963 remote=/127.0.0.1:49240]. Total timeout mills is 60000, 59614 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:23:07,275 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:50076 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:41523:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50076 dst: /127.0.0.1:41523 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:23:07,275 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:58808 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:37811:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58808 dst: /127.0.0.1:37811 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:23:07,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741917_1101 (size=1659) 2024-12-09T13:23:07,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741917_1101 (size=1659) 2024-12-09T13:23:07,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741917_1101 (size=1659) 2024-12-09T13:23:07,499 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-09T13:23:07,499 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-09T13:23:07,500 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-09T13:23:07,500 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-09T13:23:08,719 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T13:23:10,940 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41625/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733750585592/wal.1733750585788 after 4002ms 2024-12-09T13:23:10,949 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41625/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733750585592/wal.1733750585788 took 4012ms 2024-12-09T13:23:10,966 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41625/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733750585592/wal.1733750585788; continuing. 2024-12-09T13:23:10,966 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41625/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733750585592/wal.1733750585788 so closing down 2024-12-09T13:23:10,967 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T13:23:10,995 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733750585788.temp 2024-12-09T13:23:11,014 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/recovered.edits/0000000000000000003-wal.1733750585788.temp 2024-12-09T13:23:11,014 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T13:23:11,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741923_1102 (size=698) 2024-12-09T13:23:11,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741923_1102 (size=698) 2024-12-09T13:23:11,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741923_1102 (size=698) 2024-12-09T13:23:11,089 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/recovered.edits/0000000000000000003-wal.1733750585788.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-09T13:23:11,103 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/recovered.edits/0000000000000000003-wal.1733750585788.temp to hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/recovered.edits/0000000000000000008 2024-12-09T13:23:11,106 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 150 ms; skipped=3; WAL=hdfs://localhost:41625/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733750585592/wal.1733750585788, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T13:23:11,106 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41625/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733750585592/wal.1733750585788, journal: Splitting hdfs://localhost:41625/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733750585592/wal.1733750585788, size=0 (0bytes) at 1733750586938Finishing writing output for hdfs://localhost:41625/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733750585592/wal.1733750585788 so closing down at 1733750590966 (+4028 ms)Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/recovered.edits/0000000000000000003-wal.1733750585788.temp at 1733750591014 (+48 ms)3 split writer threads finished at 1733750591014Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/recovered.edits/0000000000000000003-wal.1733750585788.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733750591089 (+75 ms)Rename recovered edits hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/recovered.edits/0000000000000000003-wal.1733750585788.temp to hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/recovered.edits/0000000000000000008 at 1733750591103 (+14 ms)Processed 5 edits across 1 Regions in 150 ms; skipped=3; WAL=hdfs://localhost:41625/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733750585592/wal.1733750585788, size=0, length=0, corrupted=false, cancelled=false at 1733750591106 (+3 ms) 2024-12-09T13:23:11,116 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41625/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733750585592/wal.1733750585788 to hdfs://localhost:41625/hbase/oldWALs/wal.1733750585788 2024-12-09T13:23:11,119 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/recovered.edits/0000000000000000008 2024-12-09T13:23:11,119 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T13:23:11,127 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733750585592, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:23:11,148 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733750585592/wal.1733750591127, exclude list is [], retry=0 2024-12-09T13:23:11,159 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:23:11,159 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:23:11,159 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:23:11,197 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733750585592/wal.1733750591127 2024-12-09T13:23:11,202 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34135:34135),(127.0.0.1/127.0.0.1:34323:34323)] 2024-12-09T13:23:11,203 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => eeff0a00bbc1a0fec494b91bb4c7c821, NAME => 'testCompactedBulkLoadedFiles,,1733750585594.eeff0a00bbc1a0fec494b91bb4c7c821.', STARTKEY => '', ENDKEY => ''} 2024-12-09T13:23:11,203 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733750585594.eeff0a00bbc1a0fec494b91bb4c7c821.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:23:11,203 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:11,203 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:11,226 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:11,243 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eeff0a00bbc1a0fec494b91bb4c7c821 columnFamilyName a 2024-12-09T13:23:11,243 DEBUG [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:11,359 DEBUG [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/1bd52b11886f47ed968a0dad0629c63a 2024-12-09T13:23:11,383 DEBUG [StoreFileOpener-eeff0a00bbc1a0fec494b91bb4c7c821-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3c03515cdb144048b6c45858c9671095_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T13:23:11,384 DEBUG [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/3c03515cdb144048b6c45858c9671095_SeqId_4_ 2024-12-09T13:23:11,423 DEBUG [StoreFileOpener-eeff0a00bbc1a0fec494b91bb4c7c821-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3e2231843afc4eb6aacfc2f3d020e739_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T13:23:11,423 DEBUG [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/3e2231843afc4eb6aacfc2f3d020e739_SeqId_4_ 2024-12-09T13:23:11,440 DEBUG [StoreFileOpener-eeff0a00bbc1a0fec494b91bb4c7c821-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for b061dccd38cc452cbe407fe0fd346be9_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T13:23:11,440 DEBUG [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/b061dccd38cc452cbe407fe0fd346be9_SeqId_4_ 2024-12-09T13:23:11,459 DEBUG [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/e6210a73856a4a9893f973549e1d7fc8 2024-12-09T13:23:11,459 WARN [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/3c03515cdb144048b6c45858c9671095_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@33e1d6d2 2024-12-09T13:23:11,459 WARN [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/3e2231843afc4eb6aacfc2f3d020e739_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@33e1d6d2 2024-12-09T13:23:11,460 WARN [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/b061dccd38cc452cbe407fe0fd346be9_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@33e1d6d2 2024-12-09T13:23:11,460 WARN [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/e6210a73856a4a9893f973549e1d7fc8 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@33e1d6d2 2024-12-09T13:23:11,460 DEBUG [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/3c03515cdb144048b6c45858c9671095_SeqId_4_, hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/3e2231843afc4eb6aacfc2f3d020e739_SeqId_4_, hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/b061dccd38cc452cbe407fe0fd346be9_SeqId_4_, hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/e6210a73856a4a9893f973549e1d7fc8] to archive 2024-12-09T13:23:11,467 DEBUG [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T13:23:11,469 DEBUG [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/3c03515cdb144048b6c45858c9671095_SeqId_4_ to hdfs://localhost:41625/hbase/archive/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/3c03515cdb144048b6c45858c9671095_SeqId_4_ 2024-12-09T13:23:11,476 DEBUG [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/3e2231843afc4eb6aacfc2f3d020e739_SeqId_4_ to hdfs://localhost:41625/hbase/archive/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/3e2231843afc4eb6aacfc2f3d020e739_SeqId_4_ 2024-12-09T13:23:11,486 DEBUG [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/b061dccd38cc452cbe407fe0fd346be9_SeqId_4_ to hdfs://localhost:41625/hbase/archive/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/b061dccd38cc452cbe407fe0fd346be9_SeqId_4_ 2024-12-09T13:23:11,495 DEBUG [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/e6210a73856a4a9893f973549e1d7fc8 to hdfs://localhost:41625/hbase/archive/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/a/e6210a73856a4a9893f973549e1d7fc8 2024-12-09T13:23:11,495 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.HStore(327): Store=eeff0a00bbc1a0fec494b91bb4c7c821/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:11,496 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:11,497 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eeff0a00bbc1a0fec494b91bb4c7c821 columnFamilyName b 2024-12-09T13:23:11,497 DEBUG [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:11,499 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.HStore(327): Store=eeff0a00bbc1a0fec494b91bb4c7c821/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:11,499 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:11,500 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eeff0a00bbc1a0fec494b91bb4c7c821 columnFamilyName c 2024-12-09T13:23:11,500 DEBUG [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:11,501 INFO [StoreOpener-eeff0a00bbc1a0fec494b91bb4c7c821-1 {}] regionserver.HStore(327): Store=eeff0a00bbc1a0fec494b91bb4c7c821/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:11,510 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:11,514 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:11,532 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:11,535 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/recovered.edits/0000000000000000008 2024-12-09T13:23:11,555 DEBUG [Time-limited test {}] regionserver.HRegion(5836): eeff0a00bbc1a0fec494b91bb4c7c821 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "eeff0a00bbc1a0fec494b91bb4c7c821" family_name: "a" compaction_input: "e6210a73856a4a9893f973549e1d7fc8" compaction_input: "3e2231843afc4eb6aacfc2f3d020e739_SeqId_4_" compaction_input: "3c03515cdb144048b6c45858c9671095_SeqId_4_" compaction_input: "b061dccd38cc452cbe407fe0fd346be9_SeqId_4_" compaction_output: "1bd52b11886f47ed968a0dad0629c63a" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733750585594.eeff0a00bbc1a0fec494b91bb4c7c821." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-09T13:23:11,555 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-09T13:23:11,555 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/recovered.edits/0000000000000000008 2024-12-09T13:23:11,564 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/recovered.edits/0000000000000000008 2024-12-09T13:23:11,574 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:11,574 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:11,578 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T13:23:11,587 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for eeff0a00bbc1a0fec494b91bb4c7c821 2024-12-09T13:23:11,604 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/hbase/data/default/testCompactedBulkLoadedFiles/eeff0a00bbc1a0fec494b91bb4c7c821/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-09T13:23:11,613 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened eeff0a00bbc1a0fec494b91bb4c7c821; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70741831, jitterRate=0.05413542687892914}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T13:23:11,613 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for eeff0a00bbc1a0fec494b91bb4c7c821: Writing region info on filesystem at 1733750591203Initializing all the Stores at 1733750591218 (+15 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750591218Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750591223 (+5 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750591223Cleaning up temporary data from old regions at 1733750591574 (+351 ms)Region opened successfully at 1733750591613 (+39 ms) 2024-12-09T13:23:11,620 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing eeff0a00bbc1a0fec494b91bb4c7c821, disabling compactions & flushes 2024-12-09T13:23:11,620 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733750585594.eeff0a00bbc1a0fec494b91bb4c7c821. 2024-12-09T13:23:11,620 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733750585594.eeff0a00bbc1a0fec494b91bb4c7c821. 2024-12-09T13:23:11,620 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733750585594.eeff0a00bbc1a0fec494b91bb4c7c821. after waiting 0 ms 2024-12-09T13:23:11,620 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733750585594.eeff0a00bbc1a0fec494b91bb4c7c821. 2024-12-09T13:23:11,636 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733750585594.eeff0a00bbc1a0fec494b91bb4c7c821. 2024-12-09T13:23:11,636 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for eeff0a00bbc1a0fec494b91bb4c7c821: Waiting for close lock at 1733750591620Disabling compacts and flushes for region at 1733750591620Disabling writes for close at 1733750591620Writing region close event to WAL at 1733750591636 (+16 ms)Closed at 1733750591636 2024-12-09T13:23:11,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741924_1103 (size=93) 2024-12-09T13:23:11,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741924_1103 (size=93) 2024-12-09T13:23:11,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741924_1103 (size=93) 2024-12-09T13:23:11,680 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T13:23:11,680 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733750591127) 2024-12-09T13:23:11,707 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=434 (was 423) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_250588985_22 at /127.0.0.1:52808 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_250588985_22 at /127.0.0.1:42204 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (587680655) connection to localhost/127.0.0.1:41625 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:41625 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_250588985_22 at /127.0.0.1:54732 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1346 (was 1268) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=678 (was 684), ProcessCount=11 (was 11), AvailableMemoryMB=3209 (was 3332) 2024-12-09T13:23:11,707 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-12-09T13:23:11,744 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=434, OpenFileDescriptor=1346, MaxFileDescriptor=1048576, SystemLoadAverage=678, ProcessCount=11, AvailableMemoryMB=3208 2024-12-09T13:23:11,744 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-12-09T13:23:11,773 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:23:11,776 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T13:23:11,777 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T13:23:11,800 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-17851374, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/hregion-17851374, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:23:11,825 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-17851374/hregion-17851374.1733750591801, exclude list is [], retry=0 2024-12-09T13:23:11,839 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:23:11,839 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:23:11,843 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:23:11,874 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-17851374/hregion-17851374.1733750591801 2024-12-09T13:23:11,878 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34323:34323),(127.0.0.1/127.0.0.1:34135:34135)] 2024-12-09T13:23:11,879 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 89f0d31f743f661cde62b3eff0fc36bf, NAME => 'testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41625/hbase 2024-12-09T13:23:11,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741926_1105 (size=67) 2024-12-09T13:23:11,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741926_1105 (size=67) 2024-12-09T13:23:11,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741926_1105 (size=67) 2024-12-09T13:23:11,975 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:23:11,990 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:12,015 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89f0d31f743f661cde62b3eff0fc36bf columnFamilyName a 2024-12-09T13:23:12,015 DEBUG [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:12,023 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.HStore(327): Store=89f0d31f743f661cde62b3eff0fc36bf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:12,023 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:12,034 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89f0d31f743f661cde62b3eff0fc36bf columnFamilyName b 2024-12-09T13:23:12,034 DEBUG [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:12,039 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.HStore(327): Store=89f0d31f743f661cde62b3eff0fc36bf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:12,039 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:12,042 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89f0d31f743f661cde62b3eff0fc36bf columnFamilyName c 2024-12-09T13:23:12,042 DEBUG [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:12,043 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.HStore(327): Store=89f0d31f743f661cde62b3eff0fc36bf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:12,050 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:12,052 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:12,052 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:12,054 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:12,054 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:12,059 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T13:23:12,063 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:12,081 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T13:23:12,082 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 89f0d31f743f661cde62b3eff0fc36bf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60863631, jitterRate=-0.09306122362613678}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T13:23:12,082 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 89f0d31f743f661cde62b3eff0fc36bf: Writing region info on filesystem at 1733750591975Initializing all the Stores at 1733750591983 (+8 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750591983Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750591989 (+6 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750591989Cleaning up temporary data from old regions at 1733750592054 (+65 ms)Region opened successfully at 1733750592082 (+28 ms) 2024-12-09T13:23:12,082 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 89f0d31f743f661cde62b3eff0fc36bf, disabling compactions & flushes 2024-12-09T13:23:12,082 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf. 2024-12-09T13:23:12,082 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf. 2024-12-09T13:23:12,082 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf. after waiting 0 ms 2024-12-09T13:23:12,082 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf. 2024-12-09T13:23:12,084 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf. 2024-12-09T13:23:12,084 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 89f0d31f743f661cde62b3eff0fc36bf: Waiting for close lock at 1733750592082Disabling compacts and flushes for region at 1733750592082Disabling writes for close at 1733750592082Writing region close event to WAL at 1733750592084 (+2 ms)Closed at 1733750592084 2024-12-09T13:23:12,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741925_1104 (size=93) 2024-12-09T13:23:12,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741925_1104 (size=93) 2024-12-09T13:23:12,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741925_1104 (size=93) 2024-12-09T13:23:12,125 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T13:23:12,125 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-17851374:(num 1733750591801) 2024-12-09T13:23:12,125 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T13:23:12,139 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:23:12,166 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592139, exclude list is [], retry=0 2024-12-09T13:23:12,170 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:23:12,170 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:23:12,170 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:23:12,200 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592139 2024-12-09T13:23:12,202 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34135:34135),(127.0.0.1/127.0.0.1:34323:34323),(127.0.0.1/127.0.0.1:44161:44161)] 2024-12-09T13:23:12,203 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 89f0d31f743f661cde62b3eff0fc36bf, NAME => 'testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf.', STARTKEY => '', ENDKEY => ''} 2024-12-09T13:23:12,203 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:23:12,203 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:12,203 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:12,214 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:12,224 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89f0d31f743f661cde62b3eff0fc36bf columnFamilyName a 2024-12-09T13:23:12,224 DEBUG [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:12,231 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.HStore(327): Store=89f0d31f743f661cde62b3eff0fc36bf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:12,231 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:12,239 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89f0d31f743f661cde62b3eff0fc36bf columnFamilyName b 2024-12-09T13:23:12,239 DEBUG [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:12,244 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.HStore(327): Store=89f0d31f743f661cde62b3eff0fc36bf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:12,244 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:12,251 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89f0d31f743f661cde62b3eff0fc36bf columnFamilyName c 2024-12-09T13:23:12,251 DEBUG [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:12,258 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.HStore(327): Store=89f0d31f743f661cde62b3eff0fc36bf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:12,259 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:12,264 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:12,271 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:12,282 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:12,282 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:12,285 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T13:23:12,300 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:12,310 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 89f0d31f743f661cde62b3eff0fc36bf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66484948, jitterRate=-0.009297072887420654}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T13:23:12,311 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 89f0d31f743f661cde62b3eff0fc36bf: Writing region info on filesystem at 1733750592203Initializing all the Stores at 1733750592211 (+8 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750592211Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750592213 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750592213Cleaning up temporary data from old regions at 1733750592282 (+69 ms)Region opened successfully at 1733750592310 (+28 ms) 2024-12-09T13:23:12,403 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 89f0d31f743f661cde62b3eff0fc36bf 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-09T13:23:12,452 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/.tmp/a/32c9a8c1a2d84234b9b9596088c184bb is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733750592311/Put/seqid=0 2024-12-09T13:23:12,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741928_1107 (size=5958) 2024-12-09T13:23:12,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741928_1107 (size=5958) 2024-12-09T13:23:12,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741928_1107 (size=5958) 2024-12-09T13:23:12,499 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/.tmp/a/32c9a8c1a2d84234b9b9596088c184bb 2024-12-09T13:23:12,530 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/.tmp/a/32c9a8c1a2d84234b9b9596088c184bb as hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/a/32c9a8c1a2d84234b9b9596088c184bb 2024-12-09T13:23:12,548 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/a/32c9a8c1a2d84234b9b9596088c184bb, entries=10, sequenceid=13, filesize=5.8 K 2024-12-09T13:23:12,551 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 89f0d31f743f661cde62b3eff0fc36bf in 148ms, sequenceid=13, compaction requested=false 2024-12-09T13:23:12,551 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 89f0d31f743f661cde62b3eff0fc36bf: 2024-12-09T13:23:12,696 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 89f0d31f743f661cde62b3eff0fc36bf, disabling compactions & flushes 2024-12-09T13:23:12,696 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf. 2024-12-09T13:23:12,696 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf. 2024-12-09T13:23:12,696 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf. after waiting 0 ms 2024-12-09T13:23:12,696 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf. 2024-12-09T13:23:12,710 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf. 2024-12-09T13:23:12,710 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf. 2024-12-09T13:23:12,711 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 89f0d31f743f661cde62b3eff0fc36bf: Waiting for close lock at 1733750592695Disabling compacts and flushes for region at 1733750592695Disabling writes for close at 1733750592696 (+1 ms)Writing region close event to WAL at 1733750592710 (+14 ms)Closed at 1733750592710 2024-12-09T13:23:12,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741927_1106 (size=5869) 2024-12-09T13:23:12,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741927_1106 (size=5869) 2024-12-09T13:23:12,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741927_1106 (size=5869) 2024-12-09T13:23:12,788 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592139, size=5.7 K (5869bytes) 2024-12-09T13:23:12,788 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592139 2024-12-09T13:23:12,789 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592139 after 1ms 2024-12-09T13:23:12,803 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592139 took 15ms 2024-12-09T13:23:12,812 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592139 so closing down 2024-12-09T13:23:12,812 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T13:23:12,823 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733750592139.temp 2024-12-09T13:23:12,825 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/0000000000000000003-wal.1733750592139.temp 2024-12-09T13:23:12,825 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T13:23:12,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741929_1108 (size=5223) 2024-12-09T13:23:12,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741929_1108 (size=5223) 2024-12-09T13:23:12,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741929_1108 (size=5223) 2024-12-09T13:23:12,880 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/0000000000000000003-wal.1733750592139.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-09T13:23:12,889 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/0000000000000000003-wal.1733750592139.temp to hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/0000000000000000035 2024-12-09T13:23:12,889 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 83 ms; skipped=2; WAL=hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592139, size=5.7 K, length=5869, corrupted=false, cancelled=false 2024-12-09T13:23:12,889 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592139, journal: Splitting hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592139, size=5.7 K (5869bytes) at 1733750592788Finishing writing output for hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592139 so closing down at 1733750592812 (+24 ms)Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/0000000000000000003-wal.1733750592139.temp at 1733750592825 (+13 ms)3 split writer threads finished at 1733750592825Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/0000000000000000003-wal.1733750592139.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733750592880 (+55 ms)Rename recovered edits hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/0000000000000000003-wal.1733750592139.temp to hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/0000000000000000035 at 1733750592889 (+9 ms)Processed 32 edits across 1 Regions in 83 ms; skipped=2; WAL=hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592139, size=5.7 K, length=5869, corrupted=false, cancelled=false at 1733750592889 2024-12-09T13:23:12,906 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592139 to hdfs://localhost:41625/hbase/oldWALs/wal.1733750592139 2024-12-09T13:23:12,914 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/0000000000000000035 2024-12-09T13:23:12,915 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T13:23:12,941 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:23:12,963 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592942, exclude list is [], retry=0 2024-12-09T13:23:12,978 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:23:12,978 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:23:12,978 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:23:13,017 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592942 2024-12-09T13:23:13,018 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34135:34135),(127.0.0.1/127.0.0.1:34323:34323)] 2024-12-09T13:23:13,019 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 89f0d31f743f661cde62b3eff0fc36bf, NAME => 'testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf.', STARTKEY => '', ENDKEY => ''} 2024-12-09T13:23:13,019 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:23:13,019 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:13,019 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:13,038 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:13,047 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89f0d31f743f661cde62b3eff0fc36bf columnFamilyName a 2024-12-09T13:23:13,047 DEBUG [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:13,099 DEBUG [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/a/32c9a8c1a2d84234b9b9596088c184bb 2024-12-09T13:23:13,099 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.HStore(327): Store=89f0d31f743f661cde62b3eff0fc36bf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:13,099 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:13,107 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89f0d31f743f661cde62b3eff0fc36bf columnFamilyName b 2024-12-09T13:23:13,107 DEBUG [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:13,114 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.HStore(327): Store=89f0d31f743f661cde62b3eff0fc36bf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:13,114 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:13,119 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89f0d31f743f661cde62b3eff0fc36bf columnFamilyName c 2024-12-09T13:23:13,119 DEBUG [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:13,130 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.HStore(327): Store=89f0d31f743f661cde62b3eff0fc36bf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:13,131 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:13,134 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:13,141 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:13,143 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/0000000000000000035 2024-12-09T13:23:13,157 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/0000000000000000035 2024-12-09T13:23:13,157 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 89f0d31f743f661cde62b3eff0fc36bf 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-09T13:23:13,164 INFO [master/cad571b91d50:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-09T13:23:13,164 INFO [master/cad571b91d50:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-09T13:23:13,202 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/.tmp/b/8cb265d68cb54a99a3abcd6a97f53bb4 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733750592551/Put/seqid=0 2024-12-09T13:23:13,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741931_1110 (size=5958) 2024-12-09T13:23:13,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741931_1110 (size=5958) 2024-12-09T13:23:13,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741931_1110 (size=5958) 2024-12-09T13:23:13,294 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/.tmp/b/8cb265d68cb54a99a3abcd6a97f53bb4 2024-12-09T13:23:13,368 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/.tmp/c/00b6d622ab3e4330a5d1e6ec66d2bdd2 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733750592603/Put/seqid=0 2024-12-09T13:23:13,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741932_1111 (size=5958) 2024-12-09T13:23:13,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741932_1111 (size=5958) 2024-12-09T13:23:13,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741932_1111 (size=5958) 2024-12-09T13:23:13,397 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/.tmp/c/00b6d622ab3e4330a5d1e6ec66d2bdd2 2024-12-09T13:23:13,427 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/.tmp/b/8cb265d68cb54a99a3abcd6a97f53bb4 as hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/b/8cb265d68cb54a99a3abcd6a97f53bb4 2024-12-09T13:23:13,447 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/b/8cb265d68cb54a99a3abcd6a97f53bb4, entries=10, sequenceid=35, filesize=5.8 K 2024-12-09T13:23:13,449 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/.tmp/c/00b6d622ab3e4330a5d1e6ec66d2bdd2 as hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/c/00b6d622ab3e4330a5d1e6ec66d2bdd2 2024-12-09T13:23:13,465 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/c/00b6d622ab3e4330a5d1e6ec66d2bdd2, entries=10, sequenceid=35, filesize=5.8 K 2024-12-09T13:23:13,465 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 89f0d31f743f661cde62b3eff0fc36bf in 308ms, sequenceid=35, compaction requested=false; wal=null 2024-12-09T13:23:13,466 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/0000000000000000035 2024-12-09T13:23:13,486 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:13,486 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:13,494 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T13:23:13,496 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:13,521 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-09T13:23:13,527 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 89f0d31f743f661cde62b3eff0fc36bf; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60802504, jitterRate=-0.0939720869064331}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T13:23:13,528 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 89f0d31f743f661cde62b3eff0fc36bf: Writing region info on filesystem at 1733750593019Initializing all the Stores at 1733750593033 (+14 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750593033Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750593034 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750593034Obtaining lock to block concurrent updates at 1733750593157 (+123 ms)Preparing flush snapshotting stores in 89f0d31f743f661cde62b3eff0fc36bf at 1733750593157Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733750593158 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf. at 1733750593158Flushing 89f0d31f743f661cde62b3eff0fc36bf/b: creating writer at 1733750593158Flushing 89f0d31f743f661cde62b3eff0fc36bf/b: appending metadata at 1733750593201 (+43 ms)Flushing 89f0d31f743f661cde62b3eff0fc36bf/b: closing flushed file at 1733750593201Flushing 89f0d31f743f661cde62b3eff0fc36bf/c: creating writer at 1733750593323 (+122 ms)Flushing 89f0d31f743f661cde62b3eff0fc36bf/c: appending metadata at 1733750593367 (+44 ms)Flushing 89f0d31f743f661cde62b3eff0fc36bf/c: closing flushed file at 1733750593367Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18e45f80: reopening flushed file at 1733750593420 (+53 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fc2a7f6: reopening flushed file at 1733750593447 (+27 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 89f0d31f743f661cde62b3eff0fc36bf in 308ms, sequenceid=35, compaction requested=false; wal=null at 1733750593465 (+18 ms)Cleaning up temporary data from old regions at 1733750593486 (+21 ms)Region opened successfully at 1733750593527 (+41 ms) 2024-12-09T13:23:13,754 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592942, size=0 (0bytes) 2024-12-09T13:23:13,754 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592942 might be still open, length is 0 2024-12-09T13:23:13,754 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592942 2024-12-09T13:23:13,764 WARN [IPC Server handler 4 on default port 41625 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592942 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-12-09T13:23:13,765 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592942 after 11ms 2024-12-09T13:23:16,276 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:42260 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:41523:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42260 dst: /127.0.0.1:41523 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41523 remote=/127.0.0.1:42260]. Total timeout mills is 60000, 57398 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:23:16,277 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:52860 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:37811:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52860 dst: /127.0.0.1:37811 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:23:16,277 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_628947446_22 at /127.0.0.1:54812 [Receiving block BP-1859740653-172.17.0.3-1733750541580:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:40963:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54812 dst: /127.0.0.1:40963 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:23:16,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741930_1112 (size=5215) 2024-12-09T13:23:16,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741930_1112 (size=5215) 2024-12-09T13:23:17,766 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592942 after 4012ms 2024-12-09T13:23:17,770 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592942 took 4016ms 2024-12-09T13:23:17,789 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592942; continuing. 2024-12-09T13:23:17,789 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592942 so closing down 2024-12-09T13:23:17,789 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T13:23:17,795 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733750592942.temp 2024-12-09T13:23:17,797 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/0000000000000000037-wal.1733750592942.temp 2024-12-09T13:23:17,797 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T13:23:17,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741933_1113 (size=5223) 2024-12-09T13:23:17,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741933_1113 (size=5223) 2024-12-09T13:23:17,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741933_1113 (size=5223) 2024-12-09T13:23:17,834 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/0000000000000000037-wal.1733750592942.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-09T13:23:17,836 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/0000000000000000037-wal.1733750592942.temp to hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/0000000000000000066 2024-12-09T13:23:17,836 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 54 ms; skipped=0; WAL=hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592942, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T13:23:17,837 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592942, journal: Splitting hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592942, size=0 (0bytes) at 1733750593754Finishing writing output for hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592942 so closing down at 1733750597789 (+4035 ms)Creating recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/0000000000000000037-wal.1733750592942.temp at 1733750597797 (+8 ms)3 split writer threads finished at 1733750597797Closed recovered edits writer path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/0000000000000000037-wal.1733750592942.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733750597835 (+38 ms)Rename recovered edits hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/0000000000000000037-wal.1733750592942.temp to hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/0000000000000000066 at 1733750597836 (+1 ms)Processed 30 edits across 1 Regions in 54 ms; skipped=0; WAL=hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592942, size=0, length=0, corrupted=false, cancelled=false at 1733750597837 (+1 ms) 2024-12-09T13:23:17,850 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592942 to hdfs://localhost:41625/hbase/oldWALs/wal.1733750592942 2024-12-09T13:23:17,854 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/0000000000000000066 2024-12-09T13:23:17,854 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T13:23:17,875 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41625/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767, archiveDir=hdfs://localhost:41625/hbase/oldWALs, maxLogs=32 2024-12-09T13:23:17,896 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750597876, exclude list is [], retry=0 2024-12-09T13:23:17,904 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40963,DS-bb1bc572-231a-4487-9b7d-89ab2125a50b,DISK] 2024-12-09T13:23:17,905 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41523,DS-693cde34-0a47-46a5-8ae7-9edd9492bc06,DISK] 2024-12-09T13:23:17,905 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37811,DS-8af8162c-427e-4d04-be85-a94deb21afc3,DISK] 2024-12-09T13:23:17,921 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750597876 2024-12-09T13:23:17,922 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34323:34323),(127.0.0.1/127.0.0.1:44161:44161),(127.0.0.1/127.0.0.1:34135:34135)] 2024-12-09T13:23:17,923 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T13:23:17,927 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:17,938 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89f0d31f743f661cde62b3eff0fc36bf columnFamilyName a 2024-12-09T13:23:17,938 DEBUG [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:17,951 DEBUG [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/a/32c9a8c1a2d84234b9b9596088c184bb 2024-12-09T13:23:17,952 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.HStore(327): Store=89f0d31f743f661cde62b3eff0fc36bf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:17,952 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:17,953 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89f0d31f743f661cde62b3eff0fc36bf columnFamilyName b 2024-12-09T13:23:17,953 DEBUG [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:17,970 DEBUG [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/b/8cb265d68cb54a99a3abcd6a97f53bb4 2024-12-09T13:23:17,970 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.HStore(327): Store=89f0d31f743f661cde62b3eff0fc36bf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:17,970 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:17,972 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89f0d31f743f661cde62b3eff0fc36bf columnFamilyName c 2024-12-09T13:23:17,972 DEBUG [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T13:23:18,006 DEBUG [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/c/00b6d622ab3e4330a5d1e6ec66d2bdd2 2024-12-09T13:23:18,006 INFO [StoreOpener-89f0d31f743f661cde62b3eff0fc36bf-1 {}] regionserver.HStore(327): Store=89f0d31f743f661cde62b3eff0fc36bf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T13:23:18,006 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:18,011 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:18,018 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:18,019 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/0000000000000000066 2024-12-09T13:23:18,047 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/0000000000000000066 2024-12-09T13:23:18,047 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 89f0d31f743f661cde62b3eff0fc36bf 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-09T13:23:18,094 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/.tmp/a/e58fa06e06504d379538d2bcba17f6fa is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733750593545/Put/seqid=0 2024-12-09T13:23:18,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741935_1115 (size=5958) 2024-12-09T13:23:18,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741935_1115 (size=5958) 2024-12-09T13:23:18,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741935_1115 (size=5958) 2024-12-09T13:23:18,127 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/.tmp/a/e58fa06e06504d379538d2bcba17f6fa 2024-12-09T13:23:18,163 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/.tmp/b/bad54536bc6d4c5ca195376df57991ec is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733750593602/Put/seqid=0 2024-12-09T13:23:18,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741936_1116 (size=5958) 2024-12-09T13:23:18,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741936_1116 (size=5958) 2024-12-09T13:23:18,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741936_1116 (size=5958) 2024-12-09T13:23:18,198 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/.tmp/b/bad54536bc6d4c5ca195376df57991ec 2024-12-09T13:23:18,282 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/.tmp/c/5ff914f9551e43799ba4f836e3d12ca2 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733750593645/Put/seqid=0 2024-12-09T13:23:18,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741937_1117 (size=5958) 2024-12-09T13:23:18,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741937_1117 (size=5958) 2024-12-09T13:23:18,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741937_1117 (size=5958) 2024-12-09T13:23:18,385 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/.tmp/c/5ff914f9551e43799ba4f836e3d12ca2 2024-12-09T13:23:18,406 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/.tmp/a/e58fa06e06504d379538d2bcba17f6fa as hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/a/e58fa06e06504d379538d2bcba17f6fa 2024-12-09T13:23:18,444 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/a/e58fa06e06504d379538d2bcba17f6fa, entries=10, sequenceid=66, filesize=5.8 K 2024-12-09T13:23:18,458 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/.tmp/b/bad54536bc6d4c5ca195376df57991ec as hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/b/bad54536bc6d4c5ca195376df57991ec 2024-12-09T13:23:18,509 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/b/bad54536bc6d4c5ca195376df57991ec, entries=10, sequenceid=66, filesize=5.8 K 2024-12-09T13:23:18,514 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/.tmp/c/5ff914f9551e43799ba4f836e3d12ca2 as hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/c/5ff914f9551e43799ba4f836e3d12ca2 2024-12-09T13:23:18,543 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/c/5ff914f9551e43799ba4f836e3d12ca2, entries=10, sequenceid=66, filesize=5.8 K 2024-12-09T13:23:18,543 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 89f0d31f743f661cde62b3eff0fc36bf in 496ms, sequenceid=66, compaction requested=false; wal=null 2024-12-09T13:23:18,547 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/0000000000000000066 2024-12-09T13:23:18,563 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:18,563 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:18,566 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T13:23:18,586 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 89f0d31f743f661cde62b3eff0fc36bf 2024-12-09T13:23:18,602 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/hbase/data/default/testReplayEditsWrittenViaHRegion/89f0d31f743f661cde62b3eff0fc36bf/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-09T13:23:18,615 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 89f0d31f743f661cde62b3eff0fc36bf; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70833332, jitterRate=0.05549889802932739}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T13:23:18,615 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 89f0d31f743f661cde62b3eff0fc36bf: Writing region info on filesystem at 1733750597923Initializing all the Stores at 1733750597924 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750597924Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750597926 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733750597926Obtaining lock to block concurrent updates at 1733750598047 (+121 ms)Preparing flush snapshotting stores in 89f0d31f743f661cde62b3eff0fc36bf at 1733750598047Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733750598048 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf. at 1733750598048Flushing 89f0d31f743f661cde62b3eff0fc36bf/a: creating writer at 1733750598048Flushing 89f0d31f743f661cde62b3eff0fc36bf/a: appending metadata at 1733750598093 (+45 ms)Flushing 89f0d31f743f661cde62b3eff0fc36bf/a: closing flushed file at 1733750598093Flushing 89f0d31f743f661cde62b3eff0fc36bf/b: creating writer at 1733750598137 (+44 ms)Flushing 89f0d31f743f661cde62b3eff0fc36bf/b: appending metadata at 1733750598163 (+26 ms)Flushing 89f0d31f743f661cde62b3eff0fc36bf/b: closing flushed file at 1733750598163Flushing 89f0d31f743f661cde62b3eff0fc36bf/c: creating writer at 1733750598226 (+63 ms)Flushing 89f0d31f743f661cde62b3eff0fc36bf/c: appending metadata at 1733750598281 (+55 ms)Flushing 89f0d31f743f661cde62b3eff0fc36bf/c: closing flushed file at 1733750598281Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@498983f6: reopening flushed file at 1733750598405 (+124 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ecba6e5: reopening flushed file at 1733750598444 (+39 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22311950: reopening flushed file at 1733750598509 (+65 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 89f0d31f743f661cde62b3eff0fc36bf in 496ms, sequenceid=66, compaction requested=false; wal=null at 1733750598544 (+35 ms)Cleaning up temporary data from old regions at 1733750598563 (+19 ms)Region opened successfully at 1733750598615 (+52 ms) 2024-12-09T13:23:18,667 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 89f0d31f743f661cde62b3eff0fc36bf, disabling compactions & flushes 2024-12-09T13:23:18,667 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf. 2024-12-09T13:23:18,667 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf. 2024-12-09T13:23:18,667 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf. after waiting 0 ms 2024-12-09T13:23:18,667 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf. 2024-12-09T13:23:18,703 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733750591774.89f0d31f743f661cde62b3eff0fc36bf. 2024-12-09T13:23:18,703 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 89f0d31f743f661cde62b3eff0fc36bf: Waiting for close lock at 1733750598667Disabling compacts and flushes for region at 1733750598667Disabling writes for close at 1733750598667Writing region close event to WAL at 1733750598703 (+36 ms)Closed at 1733750598703 2024-12-09T13:23:18,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741934_1114 (size=93) 2024-12-09T13:23:18,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741934_1114 (size=93) 2024-12-09T13:23:18,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741934_1114 (size=93) 2024-12-09T13:23:18,758 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T13:23:18,759 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733750597876) 2024-12-09T13:23:18,784 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=438 (was 434) Potentially hanging thread: IPC Client (587680655) connection to localhost/127.0.0.1:41625 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:41625 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_533033455_22 at /127.0.0.1:42314 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_533033455_22 at /127.0.0.1:49784 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_533033455_22 at /127.0.0.1:54834 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1408 (was 1346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=820 (was 678) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3199 (was 3208) 2024-12-09T13:23:18,784 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1408 is superior to 1024 2024-12-09T13:23:18,784 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-09T13:23:18,784 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T13:23:18,784 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T13:23:18,784 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T13:23:18,790 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T13:23:18,790 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T13:23:18,790 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T13:23:18,791 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1884317825, stopped=false 2024-12-09T13:23:18,791 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=cad571b91d50,44823,1733750546819 2024-12-09T13:23:18,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T13:23:18,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34317-0x1012b596d9a0003, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T13:23:18,793 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T13:23:18,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:23:18,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39675-0x1012b596d9a0001, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T13:23:18,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34317-0x1012b596d9a0003, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:23:18,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39675-0x1012b596d9a0001, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:23:18,794 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39675-0x1012b596d9a0001, quorum=127.0.0.1:53371, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T13:23:18,795 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T13:23:18,795 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T13:23:18,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T13:23:18,795 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'cad571b91d50,39675,1733750548162' ***** 2024-12-09T13:23:18,795 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T13:23:18,795 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'cad571b91d50,34317,1733750548453' ***** 2024-12-09T13:23:18,795 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T13:23:18,795 INFO [RS:0;cad571b91d50:39675 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T13:23:18,795 INFO [RS:2;cad571b91d50:34317 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T13:23:18,796 INFO [RS:2;cad571b91d50:34317 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T13:23:18,796 INFO [RS:0;cad571b91d50:39675 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T13:23:18,796 INFO [RS:0;cad571b91d50:39675 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T13:23:18,796 INFO [RS:2;cad571b91d50:34317 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T13:23:18,796 INFO [RS:2;cad571b91d50:34317 {}] regionserver.HRegionServer(959): stopping server cad571b91d50,34317,1733750548453 2024-12-09T13:23:18,796 INFO [RS:2;cad571b91d50:34317 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T13:23:18,796 INFO [RS:2;cad571b91d50:34317 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;cad571b91d50:34317. 2024-12-09T13:23:18,796 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T13:23:18,796 DEBUG [RS:2;cad571b91d50:34317 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T13:23:18,796 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34317-0x1012b596d9a0003, quorum=127.0.0.1:53371, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T13:23:18,796 DEBUG [RS:2;cad571b91d50:34317 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T13:23:18,796 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T13:23:18,796 INFO [RS:2;cad571b91d50:34317 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T13:23:18,796 INFO [RS:2;cad571b91d50:34317 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T13:23:18,796 INFO [RS:2;cad571b91d50:34317 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T13:23:18,796 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T13:23:18,796 INFO [RS:2;cad571b91d50:34317 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T13:23:18,797 INFO [RS:0;cad571b91d50:39675 {}] regionserver.HRegionServer(3091): Received CLOSE for f4be257e1d5e695afc605404060499d5 2024-12-09T13:23:18,797 INFO [RS:2;cad571b91d50:34317 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T13:23:18,797 DEBUG [RS:2;cad571b91d50:34317 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-09T13:23:18,797 INFO [RS:0;cad571b91d50:39675 {}] regionserver.HRegionServer(959): stopping server cad571b91d50,39675,1733750548162 2024-12-09T13:23:18,797 INFO [RS:0;cad571b91d50:39675 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T13:23:18,797 INFO [RS:0;cad571b91d50:39675 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;cad571b91d50:39675. 2024-12-09T13:23:18,797 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f4be257e1d5e695afc605404060499d5, disabling compactions & flushes 2024-12-09T13:23:18,797 DEBUG [RS:0;cad571b91d50:39675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T13:23:18,797 INFO [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:23:18,797 DEBUG [RS:0;cad571b91d50:39675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T13:23:18,797 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:23:18,797 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. after waiting 0 ms 2024-12-09T13:23:18,797 INFO [RS:0;cad571b91d50:39675 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T13:23:18,797 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:23:18,797 DEBUG [RS:0;cad571b91d50:39675 {}] regionserver.HRegionServer(1325): Online Regions={f4be257e1d5e695afc605404060499d5=testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5.} 2024-12-09T13:23:18,802 DEBUG [RS:2;cad571b91d50:34317 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T13:23:18,806 DEBUG [RS:0;cad571b91d50:39675 {}] regionserver.HRegionServer(1351): Waiting on f4be257e1d5e695afc605404060499d5 2024-12-09T13:23:18,807 DEBUG [RS_CLOSE_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T13:23:18,807 INFO [RS_CLOSE_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T13:23:18,807 DEBUG [RS_CLOSE_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T13:23:18,807 DEBUG [RS_CLOSE_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T13:23:18,807 DEBUG [RS_CLOSE_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T13:23:18,807 INFO [RS_CLOSE_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-12-09T13:23:18,862 INFO [regionserver/cad571b91d50:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T13:23:18,862 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f4be257e1d5e695afc605404060499d5/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-09T13:23:18,870 INFO [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:23:18,870 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f4be257e1d5e695afc605404060499d5: Waiting for close lock at 1733750598797Running coprocessor pre-close hooks at 1733750598797Disabling compacts and flushes for region at 1733750598797Disabling writes for close at 1733750598797Writing region close event to WAL at 1733750598802 (+5 ms)Running coprocessor post-close hooks at 1733750598870 (+68 ms)Closed at 1733750598870 2024-12-09T13:23:18,871 DEBUG [RS_CLOSE_REGION-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5. 2024-12-09T13:23:18,877 INFO [regionserver/cad571b91d50:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T13:23:18,884 DEBUG [RS_CLOSE_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/hbase/meta/1588230740/.tmp/info/99c602d4493f42298f22c3e2fa19f3ff is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733750569196.f4be257e1d5e695afc605404060499d5./info:regioninfo/1733750572261/Put/seqid=0 2024-12-09T13:23:18,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741938_1118 (size=8243) 2024-12-09T13:23:18,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741938_1118 (size=8243) 2024-12-09T13:23:18,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741938_1118 (size=8243) 2024-12-09T13:23:18,980 INFO [RS_CLOSE_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/hbase/meta/1588230740/.tmp/info/99c602d4493f42298f22c3e2fa19f3ff 2024-12-09T13:23:19,003 DEBUG [RS:2;cad571b91d50:34317 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T13:23:19,007 INFO [RS:0;cad571b91d50:39675 {}] regionserver.HRegionServer(976): stopping server cad571b91d50,39675,1733750548162; all regions closed. 2024-12-09T13:23:19,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741835_1011 (size=3433) 2024-12-09T13:23:19,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741835_1011 (size=3433) 2024-12-09T13:23:19,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741835_1011 (size=3433) 2024-12-09T13:23:19,045 DEBUG [RS_CLOSE_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/hbase/meta/1588230740/.tmp/ns/19159ea684ca49e68b3084ca057eacb6 is 43, key is default/ns:d/1733750552643/Put/seqid=0 2024-12-09T13:23:19,078 DEBUG [RS:0;cad571b91d50:39675 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/oldWALs 2024-12-09T13:23:19,078 INFO [RS:0;cad571b91d50:39675 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL cad571b91d50%2C39675%2C1733750548162:(num 1733750551864) 2024-12-09T13:23:19,078 DEBUG [RS:0;cad571b91d50:39675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T13:23:19,079 INFO [RS:0;cad571b91d50:39675 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T13:23:19,079 INFO [RS:0;cad571b91d50:39675 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T13:23:19,079 INFO [RS:0;cad571b91d50:39675 {}] hbase.ChoreService(370): Chore service for: regionserver/cad571b91d50:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-09T13:23:19,079 INFO [RS:0;cad571b91d50:39675 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T13:23:19,079 INFO [RS:0;cad571b91d50:39675 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T13:23:19,079 INFO [RS:0;cad571b91d50:39675 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T13:23:19,079 INFO [RS:0;cad571b91d50:39675 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T13:23:19,079 INFO [RS:0;cad571b91d50:39675 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39675 2024-12-09T13:23:19,080 INFO [regionserver/cad571b91d50:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T13:23:19,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39675-0x1012b596d9a0001, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/cad571b91d50,39675,1733750548162 2024-12-09T13:23:19,084 INFO [RS:0;cad571b91d50:39675 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T13:23:19,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T13:23:19,098 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [cad571b91d50,39675,1733750548162] 2024-12-09T13:23:19,118 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/cad571b91d50,39675,1733750548162 already deleted, retry=false 2024-12-09T13:23:19,118 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; cad571b91d50,39675,1733750548162 expired; onlineServers=1 2024-12-09T13:23:19,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741939_1119 (size=5153) 2024-12-09T13:23:19,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741939_1119 (size=5153) 2024-12-09T13:23:19,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741939_1119 (size=5153) 2024-12-09T13:23:19,141 INFO [RS_CLOSE_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/hbase/meta/1588230740/.tmp/ns/19159ea684ca49e68b3084ca057eacb6 2024-12-09T13:23:19,182 DEBUG [RS_CLOSE_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/hbase/meta/1588230740/.tmp/table/a157909bcbf04dec916c6446b38fc2e8 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733750569670/Put/seqid=0 2024-12-09T13:23:19,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39675-0x1012b596d9a0001, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T13:23:19,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39675-0x1012b596d9a0001, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T13:23:19,195 INFO [RS:0;cad571b91d50:39675 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T13:23:19,195 INFO [RS:0;cad571b91d50:39675 {}] regionserver.HRegionServer(1031): Exiting; stopping=cad571b91d50,39675,1733750548162; zookeeper connection closed. 2024-12-09T13:23:19,206 DEBUG [RS:2;cad571b91d50:34317 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T13:23:19,224 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@258968b5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@258968b5 2024-12-09T13:23:19,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741940_1120 (size=5431) 2024-12-09T13:23:19,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741940_1120 (size=5431) 2024-12-09T13:23:19,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741940_1120 (size=5431) 2024-12-09T13:23:19,287 INFO [RS_CLOSE_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/hbase/meta/1588230740/.tmp/table/a157909bcbf04dec916c6446b38fc2e8 2024-12-09T13:23:19,331 DEBUG [RS_CLOSE_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/hbase/meta/1588230740/.tmp/info/99c602d4493f42298f22c3e2fa19f3ff as hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/hbase/meta/1588230740/info/99c602d4493f42298f22c3e2fa19f3ff 2024-12-09T13:23:19,345 INFO [RS_CLOSE_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/hbase/meta/1588230740/info/99c602d4493f42298f22c3e2fa19f3ff, entries=18, sequenceid=21, filesize=8.0 K 2024-12-09T13:23:19,350 DEBUG [RS_CLOSE_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/hbase/meta/1588230740/.tmp/ns/19159ea684ca49e68b3084ca057eacb6 as hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/hbase/meta/1588230740/ns/19159ea684ca49e68b3084ca057eacb6 2024-12-09T13:23:19,368 INFO [RS_CLOSE_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/hbase/meta/1588230740/ns/19159ea684ca49e68b3084ca057eacb6, entries=2, sequenceid=21, filesize=5.0 K 2024-12-09T13:23:19,370 DEBUG [RS_CLOSE_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/hbase/meta/1588230740/.tmp/table/a157909bcbf04dec916c6446b38fc2e8 as hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/hbase/meta/1588230740/table/a157909bcbf04dec916c6446b38fc2e8 2024-12-09T13:23:19,392 INFO [RS_CLOSE_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/hbase/meta/1588230740/table/a157909bcbf04dec916c6446b38fc2e8, entries=2, sequenceid=21, filesize=5.3 K 2024-12-09T13:23:19,400 INFO [RS_CLOSE_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 592ms, sequenceid=21, compaction requested=false 2024-12-09T13:23:19,410 DEBUG [RS:2;cad571b91d50:34317 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T13:23:19,447 DEBUG [RS_CLOSE_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-12-09T13:23:19,448 DEBUG [RS_CLOSE_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T13:23:19,448 INFO [RS_CLOSE_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T13:23:19,448 DEBUG [RS_CLOSE_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733750598807Running coprocessor pre-close hooks at 1733750598807Disabling compacts and flushes for region at 1733750598807Disabling writes for close at 1733750598807Obtaining lock to block concurrent updates at 1733750598807Preparing flush snapshotting stores in 1588230740 at 1733750598807Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1733750598808 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733750598817 (+9 ms)Flushing 1588230740/info: creating writer at 1733750598818 (+1 ms)Flushing 1588230740/info: appending metadata at 1733750598883 (+65 ms)Flushing 1588230740/info: closing flushed file at 1733750598883Flushing 1588230740/ns: creating writer at 1733750599010 (+127 ms)Flushing 1588230740/ns: appending metadata at 1733750599044 (+34 ms)Flushing 1588230740/ns: closing flushed file at 1733750599044Flushing 1588230740/table: creating writer at 1733750599159 (+115 ms)Flushing 1588230740/table: appending metadata at 1733750599181 (+22 ms)Flushing 1588230740/table: closing flushed file at 1733750599181Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e4bd766: reopening flushed file at 1733750599330 (+149 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@476b893f: reopening flushed file at 1733750599345 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c62efb: reopening flushed file at 1733750599368 (+23 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 592ms, sequenceid=21, compaction requested=false at 1733750599400 (+32 ms)Writing region close event to WAL at 1733750599422 (+22 ms)Running coprocessor post-close hooks at 1733750599448 (+26 ms)Closed at 1733750599448 2024-12-09T13:23:19,448 DEBUG [RS_CLOSE_META-regionserver/cad571b91d50:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T13:23:19,478 INFO [regionserver/cad571b91d50:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-09T13:23:19,478 INFO [regionserver/cad571b91d50:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-09T13:23:19,611 INFO [RS:2;cad571b91d50:34317 {}] regionserver.HRegionServer(976): stopping server cad571b91d50,34317,1733750548453; all regions closed. 2024-12-09T13:23:19,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741836_1012 (size=8899) 2024-12-09T13:23:19,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741836_1012 (size=8899) 2024-12-09T13:23:19,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741836_1012 (size=8899) 2024-12-09T13:23:19,643 DEBUG [RS:2;cad571b91d50:34317 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/oldWALs 2024-12-09T13:23:19,643 INFO [RS:2;cad571b91d50:34317 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL cad571b91d50%2C34317%2C1733750548453.meta:.meta(num 1733750552410) 2024-12-09T13:23:19,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741834_1010 (size=93) 2024-12-09T13:23:19,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741834_1010 (size=93) 2024-12-09T13:23:19,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741834_1010 (size=93) 2024-12-09T13:23:19,680 DEBUG [RS:2;cad571b91d50:34317 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/oldWALs 2024-12-09T13:23:19,680 INFO [RS:2;cad571b91d50:34317 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL cad571b91d50%2C34317%2C1733750548453:(num 1733750551832) 2024-12-09T13:23:19,681 DEBUG [RS:2;cad571b91d50:34317 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T13:23:19,681 INFO [RS:2;cad571b91d50:34317 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T13:23:19,681 INFO [RS:2;cad571b91d50:34317 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T13:23:19,681 INFO [RS:2;cad571b91d50:34317 {}] hbase.ChoreService(370): Chore service for: regionserver/cad571b91d50:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T13:23:19,681 INFO [RS:2;cad571b91d50:34317 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T13:23:19,681 INFO [RS:2;cad571b91d50:34317 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34317 2024-12-09T13:23:19,682 INFO [regionserver/cad571b91d50:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T13:23:19,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T13:23:19,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34317-0x1012b596d9a0003, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/cad571b91d50,34317,1733750548453 2024-12-09T13:23:19,684 INFO [RS:2;cad571b91d50:34317 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T13:23:19,685 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [cad571b91d50,34317,1733750548453] 2024-12-09T13:23:19,692 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/cad571b91d50,34317,1733750548453 already deleted, retry=false 2024-12-09T13:23:19,692 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; cad571b91d50,34317,1733750548453 expired; onlineServers=0 2024-12-09T13:23:19,693 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'cad571b91d50,44823,1733750546819' ***** 2024-12-09T13:23:19,693 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-09T13:23:19,693 INFO [M:0;cad571b91d50:44823 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T13:23:19,693 INFO [M:0;cad571b91d50:44823 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T13:23:19,693 DEBUG [M:0;cad571b91d50:44823 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T13:23:19,693 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T13:23:19,693 DEBUG [M:0;cad571b91d50:44823 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T13:23:19,693 DEBUG [master/cad571b91d50:0:becomeActiveMaster-HFileCleaner.large.0-1733750551077 {}] cleaner.HFileCleaner(306): Exit Thread[master/cad571b91d50:0:becomeActiveMaster-HFileCleaner.large.0-1733750551077,5,FailOnTimeoutGroup] 2024-12-09T13:23:19,693 DEBUG [master/cad571b91d50:0:becomeActiveMaster-HFileCleaner.small.0-1733750551087 {}] cleaner.HFileCleaner(306): Exit Thread[master/cad571b91d50:0:becomeActiveMaster-HFileCleaner.small.0-1733750551087,5,FailOnTimeoutGroup] 2024-12-09T13:23:19,694 INFO [M:0;cad571b91d50:44823 {}] hbase.ChoreService(370): Chore service for: master/cad571b91d50:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-09T13:23:19,694 INFO [M:0;cad571b91d50:44823 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T13:23:19,694 DEBUG [M:0;cad571b91d50:44823 {}] master.HMaster(1795): Stopping service threads 2024-12-09T13:23:19,694 INFO [M:0;cad571b91d50:44823 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T13:23:19,694 INFO [M:0;cad571b91d50:44823 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T13:23:19,695 INFO [M:0;cad571b91d50:44823 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T13:23:19,695 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T13:23:19,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T13:23:19,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T13:23:19,697 DEBUG [M:0;cad571b91d50:44823 {}] zookeeper.ZKUtil(347): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-09T13:23:19,697 WARN [M:0;cad571b91d50:44823 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-09T13:23:19,698 INFO [M:0;cad571b91d50:44823 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/.lastflushedseqids 2024-12-09T13:23:19,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741941_1121 (size=138) 2024-12-09T13:23:19,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741941_1121 (size=138) 2024-12-09T13:23:19,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741941_1121 (size=138) 2024-12-09T13:23:19,744 INFO [M:0;cad571b91d50:44823 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T13:23:19,744 INFO [M:0;cad571b91d50:44823 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T13:23:19,744 DEBUG [M:0;cad571b91d50:44823 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T13:23:19,744 INFO [M:0;cad571b91d50:44823 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T13:23:19,744 DEBUG [M:0;cad571b91d50:44823 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T13:23:19,744 DEBUG [M:0;cad571b91d50:44823 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T13:23:19,744 DEBUG [M:0;cad571b91d50:44823 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T13:23:19,745 INFO [M:0;cad571b91d50:44823 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.36 KB heapSize=83.73 KB 2024-12-09T13:23:19,767 DEBUG [M:0;cad571b91d50:44823 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d15ee3f7268e4793b334a365eb92b60d is 82, key is hbase:meta,,1/info:regioninfo/1733750552547/Put/seqid=0 2024-12-09T13:23:19,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34317-0x1012b596d9a0003, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T13:23:19,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34317-0x1012b596d9a0003, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T13:23:19,792 INFO [RS:2;cad571b91d50:34317 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T13:23:19,792 INFO [RS:2;cad571b91d50:34317 {}] regionserver.HRegionServer(1031): Exiting; stopping=cad571b91d50,34317,1733750548453; zookeeper connection closed. 2024-12-09T13:23:19,802 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2fa2bb19 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2fa2bb19 2024-12-09T13:23:19,809 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-09T13:23:19,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741942_1122 (size=5672) 2024-12-09T13:23:19,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741942_1122 (size=5672) 2024-12-09T13:23:19,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741942_1122 (size=5672) 2024-12-09T13:23:19,832 INFO [M:0;cad571b91d50:44823 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d15ee3f7268e4793b334a365eb92b60d 2024-12-09T13:23:19,866 DEBUG [M:0;cad571b91d50:44823 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/31e9bf25f48c4ca4b2ab3560b19f5a73 is 1077, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733750569679/Put/seqid=0 2024-12-09T13:23:19,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741943_1123 (size=7756) 2024-12-09T13:23:19,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741943_1123 (size=7756) 2024-12-09T13:23:19,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741943_1123 (size=7756) 2024-12-09T13:23:19,966 INFO [M:0;cad571b91d50:44823 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.62 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/31e9bf25f48c4ca4b2ab3560b19f5a73 2024-12-09T13:23:20,019 INFO [M:0;cad571b91d50:44823 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 31e9bf25f48c4ca4b2ab3560b19f5a73 2024-12-09T13:23:20,096 DEBUG [M:0;cad571b91d50:44823 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c6659416f1ad43eeaef84c172980cd4e is 69, key is cad571b91d50,34317,1733750548453/rs:state/1733750551217/Put/seqid=0 2024-12-09T13:23:20,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741944_1124 (size=5445) 2024-12-09T13:23:20,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741944_1124 (size=5445) 2024-12-09T13:23:20,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741944_1124 (size=5445) 2024-12-09T13:23:20,199 INFO [M:0;cad571b91d50:44823 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c6659416f1ad43eeaef84c172980cd4e 2024-12-09T13:23:20,225 INFO [M:0;cad571b91d50:44823 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c6659416f1ad43eeaef84c172980cd4e 2024-12-09T13:23:20,231 DEBUG [M:0;cad571b91d50:44823 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d15ee3f7268e4793b334a365eb92b60d as hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d15ee3f7268e4793b334a365eb92b60d 2024-12-09T13:23:20,237 INFO [M:0;cad571b91d50:44823 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d15ee3f7268e4793b334a365eb92b60d, entries=8, sequenceid=168, filesize=5.5 K 2024-12-09T13:23:20,239 DEBUG [M:0;cad571b91d50:44823 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/31e9bf25f48c4ca4b2ab3560b19f5a73 as hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/31e9bf25f48c4ca4b2ab3560b19f5a73 2024-12-09T13:23:20,258 INFO [M:0;cad571b91d50:44823 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 31e9bf25f48c4ca4b2ab3560b19f5a73 2024-12-09T13:23:20,258 INFO [M:0;cad571b91d50:44823 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/31e9bf25f48c4ca4b2ab3560b19f5a73, entries=17, sequenceid=168, filesize=7.6 K 2024-12-09T13:23:20,262 DEBUG [M:0;cad571b91d50:44823 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c6659416f1ad43eeaef84c172980cd4e as hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c6659416f1ad43eeaef84c172980cd4e 2024-12-09T13:23:20,287 INFO [M:0;cad571b91d50:44823 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c6659416f1ad43eeaef84c172980cd4e 2024-12-09T13:23:20,287 INFO [M:0;cad571b91d50:44823 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41625/user/jenkins/test-data/f90a0d44-57e3-7fdc-7d99-1f7701810ce1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c6659416f1ad43eeaef84c172980cd4e, entries=3, sequenceid=168, filesize=5.3 K 2024-12-09T13:23:20,295 INFO [M:0;cad571b91d50:44823 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.36 KB/69996, heapSize ~83.44 KB/85440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 551ms, sequenceid=168, compaction requested=false 2024-12-09T13:23:20,314 INFO [M:0;cad571b91d50:44823 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T13:23:20,314 DEBUG [M:0;cad571b91d50:44823 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733750599744Disabling compacts and flushes for region at 1733750599744Disabling writes for close at 1733750599744Obtaining lock to block concurrent updates at 1733750599745 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733750599745Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69996, getHeapSize=85680, getOffHeapSize=0, getCellsCount=195 at 1733750599745Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733750599746 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733750599746Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733750599767 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733750599767Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733750599843 (+76 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733750599865 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733750599865Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733750600019 (+154 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733750600095 (+76 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733750600095Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34adf3de: reopening flushed file at 1733750600225 (+130 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d78bd15: reopening flushed file at 1733750600237 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22504252: reopening flushed file at 1733750600258 (+21 ms)Finished flush of dataSize ~68.36 KB/69996, heapSize ~83.44 KB/85440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 551ms, sequenceid=168, compaction requested=false at 1733750600295 (+37 ms)Writing region close event to WAL at 1733750600314 (+19 ms)Closed at 1733750600314 2024-12-09T13:23:20,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40963 is added to blk_1073741830_1006 (size=81443) 2024-12-09T13:23:20,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37811 is added to blk_1073741830_1006 (size=81443) 2024-12-09T13:23:20,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41523 is added to blk_1073741830_1006 (size=81443) 2024-12-09T13:23:20,352 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T13:23:20,352 INFO [M:0;cad571b91d50:44823 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-09T13:23:20,353 INFO [M:0;cad571b91d50:44823 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44823 2024-12-09T13:23:20,353 INFO [M:0;cad571b91d50:44823 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T13:23:20,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T13:23:20,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44823-0x1012b596d9a0000, quorum=127.0.0.1:53371, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T13:23:20,456 INFO [M:0;cad571b91d50:44823 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T13:23:20,465 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733750572635/wal.1733750573438 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733750572635/wal.1733750573438 (inode 16655) Holder DFSClient_NONMAPREDUCE_628947446_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733750572635/wal.1733750573438 (inode 16655) Holder DFSClient_NONMAPREDUCE_628947446_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-09T13:23:20,467 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733750564414/wal.1733750564582 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:23:20,467 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733750591767/wal.1733750592942 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:23:20,476 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733750573573/wal.1733750583461 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733750573573/wal.1733750583461 (inode 16678) Holder DFSClient_NONMAPREDUCE_628947446_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733750573573/wal.1733750583461 (inode 16678) Holder DFSClient_NONMAPREDUCE_628947446_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-09T13:23:20,477 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733750554806/wal.1733750554948 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:23:20,479 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733750584385/wal.1733750585265 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733750584385/wal.1733750585265 (inode 16704) Holder DFSClient_NONMAPREDUCE_628947446_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733750584385/wal.1733750585265 (inode 16704) Holder DFSClient_NONMAPREDUCE_628947446_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-09T13:23:20,480 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733750585592/wal.1733750585788 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T13:23:20,482 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223/wal.1733750554599 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223/wal.1733750554599 (inode 16485) Holder DFSClient_NONMAPREDUCE_628947446_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733750554223/wal.1733750554599 (inode 16485) Holder DFSClient_NONMAPREDUCE_628947446_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-09T13:23:20,486 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982/wal.1733750553832 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982/wal.1733750553832 (inode 16462) Holder DFSClient_NONMAPREDUCE_628947446_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733750552982/wal.1733750553832 (inode 16462) Holder DFSClient_NONMAPREDUCE_628947446_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-09T13:23:20,500 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@49d6f087{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T13:23:20,502 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@245e172d{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T13:23:20,503 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T13:23:20,503 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@137df924{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T13:23:20,503 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@51314a16{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/hadoop.log.dir/,STOPPED} 2024-12-09T13:23:20,510 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T13:23:20,510 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T13:23:20,512 WARN [BP-1859740653-172.17.0.3-1733750541580 heartbeating to localhost/127.0.0.1:41625 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T13:23:20,513 WARN [BP-1859740653-172.17.0.3-1733750541580 heartbeating to localhost/127.0.0.1:41625 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1859740653-172.17.0.3-1733750541580 (Datanode Uuid 51e845fd-b4eb-4769-b479-94aa58893f5e) service to localhost/127.0.0.1:41625 2024-12-09T13:23:20,514 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/cluster_b6088222-ea39-2e85-344a-dadc86871089/data/data5/current/BP-1859740653-172.17.0.3-1733750541580 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T13:23:20,518 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/cluster_b6088222-ea39-2e85-344a-dadc86871089/data/data6/current/BP-1859740653-172.17.0.3-1733750541580 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T13:23:20,519 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T13:23:20,558 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@d332c17{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T13:23:20,560 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1d2ea2b6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T13:23:20,560 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T13:23:20,560 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@17f0ed67{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T13:23:20,560 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1c37dc51{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/hadoop.log.dir/,STOPPED} 2024-12-09T13:23:20,581 WARN [BP-1859740653-172.17.0.3-1733750541580 heartbeating to localhost/127.0.0.1:41625 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T13:23:20,581 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T13:23:20,581 WARN [BP-1859740653-172.17.0.3-1733750541580 heartbeating to localhost/127.0.0.1:41625 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1859740653-172.17.0.3-1733750541580 (Datanode Uuid 2cd4bdd8-71e6-4810-a8b4-c0ced52b755f) service to localhost/127.0.0.1:41625 2024-12-09T13:23:20,581 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T13:23:20,582 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/cluster_b6088222-ea39-2e85-344a-dadc86871089/data/data3/current/BP-1859740653-172.17.0.3-1733750541580 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T13:23:20,583 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T13:23:20,586 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/cluster_b6088222-ea39-2e85-344a-dadc86871089/data/data4/current/BP-1859740653-172.17.0.3-1733750541580 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T13:23:20,609 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3683c385{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T13:23:20,623 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@e143a86{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T13:23:20,623 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T13:23:20,623 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@f666beb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T13:23:20,623 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1acb58f2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/hadoop.log.dir/,STOPPED} 2024-12-09T13:23:20,648 WARN [BP-1859740653-172.17.0.3-1733750541580 heartbeating to localhost/127.0.0.1:41625 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T13:23:20,648 WARN [BP-1859740653-172.17.0.3-1733750541580 heartbeating to localhost/127.0.0.1:41625 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1859740653-172.17.0.3-1733750541580 (Datanode Uuid fce0c195-64e1-4f70-bc11-09e129f90579) service to localhost/127.0.0.1:41625 2024-12-09T13:23:20,650 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T13:23:20,650 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T13:23:20,651 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/cluster_b6088222-ea39-2e85-344a-dadc86871089/data/data2/current/BP-1859740653-172.17.0.3-1733750541580 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T13:23:20,652 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T13:23:20,654 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/cluster_b6088222-ea39-2e85-344a-dadc86871089/data/data1/current/BP-1859740653-172.17.0.3-1733750541580 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T13:23:20,671 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7fe9bfd3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T13:23:20,674 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@16695a38{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T13:23:20,674 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T13:23:20,674 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d655053{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T13:23:20,674 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b5fa2d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c4618a14-c2aa-7312-c7b5-688c8a1d0a0d/hadoop.log.dir/,STOPPED} 2024-12-09T13:23:20,697 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-09T13:23:20,793 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down