2024-12-09 16:30:33,691 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-09 16:30:33,703 main DEBUG Took 0.009778 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 16:30:33,703 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 16:30:33,704 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 16:30:33,705 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 16:30:33,706 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 16:30:33,714 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 16:30:33,728 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 16:30:33,730 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 16:30:33,730 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 16:30:33,731 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 16:30:33,731 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 16:30:33,731 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 16:30:33,732 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 16:30:33,733 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 16:30:33,733 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 16:30:33,733 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 16:30:33,734 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 16:30:33,734 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 16:30:33,735 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 16:30:33,735 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 16:30:33,735 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 16:30:33,736 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 16:30:33,736 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 16:30:33,736 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 16:30:33,737 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 16:30:33,737 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 16:30:33,737 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 16:30:33,738 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 16:30:33,738 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 16:30:33,738 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 16:30:33,739 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 16:30:33,739 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 16:30:33,740 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 16:30:33,742 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 16:30:33,744 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 16:30:33,745 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 16:30:33,746 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 16:30:33,746 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 16:30:33,755 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 16:30:33,758 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 16:30:33,760 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 16:30:33,760 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 16:30:33,761 main DEBUG createAppenders(={Console}) 2024-12-09 16:30:33,762 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-09 16:30:33,762 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-09 16:30:33,762 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-09 16:30:33,763 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 16:30:33,763 main DEBUG OutputStream closed 2024-12-09 16:30:33,763 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 16:30:33,764 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 16:30:33,764 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-09 16:30:33,837 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 16:30:33,839 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 16:30:33,841 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 16:30:33,842 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 16:30:33,843 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 16:30:33,843 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 16:30:33,843 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 16:30:33,844 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 16:30:33,844 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 16:30:33,844 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 16:30:33,845 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 16:30:33,845 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 16:30:33,845 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 16:30:33,846 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 16:30:33,846 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 16:30:33,846 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 16:30:33,846 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 16:30:33,847 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 16:30:33,849 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 16:30:33,850 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-09 16:30:33,850 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 16:30:33,850 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-09T16:30:34,140 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52 2024-12-09 16:30:34,144 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 16:30:34,144 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T16:30:34,155 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-09T16:30:34,162 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayCompressed timeout: 13 mins 2024-12-09T16:30:34,185 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-09T16:30:34,245 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-09T16:30:34,245 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-09T16:30:34,261 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-09T16:30:34,280 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/cluster_cd98e670-d246-b6f3-ff89-5ca1891baf9f, deleteOnExit=true 2024-12-09T16:30:34,280 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-09T16:30:34,281 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/test.cache.data in system properties and HBase conf 2024-12-09T16:30:34,282 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/hadoop.tmp.dir in system properties and HBase conf 2024-12-09T16:30:34,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/hadoop.log.dir in system properties and HBase conf 2024-12-09T16:30:34,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-09T16:30:34,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-09T16:30:34,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-09T16:30:34,379 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T16:30:34,496 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T16:30:34,500 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-09T16:30:34,501 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-09T16:30:34,502 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-09T16:30:34,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T16:30:34,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-09T16:30:34,504 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-09T16:30:34,504 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T16:30:34,505 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T16:30:34,505 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-09T16:30:34,506 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/nfs.dump.dir in system properties and HBase conf 2024-12-09T16:30:34,506 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/java.io.tmpdir in system properties and HBase conf 2024-12-09T16:30:34,507 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T16:30:34,507 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-09T16:30:34,507 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-09T16:30:35,508 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-09T16:30:35,607 INFO [Time-limited test {}] log.Log(170): Logging initialized @2657ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-09T16:30:35,708 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T16:30:35,785 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T16:30:35,815 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T16:30:35,815 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T16:30:35,816 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T16:30:35,840 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T16:30:35,844 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6630be9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/hadoop.log.dir/,AVAILABLE} 2024-12-09T16:30:35,846 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70596a48{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T16:30:36,070 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1843526c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/java.io.tmpdir/jetty-localhost-43967-hadoop-hdfs-3_4_1-tests_jar-_-any-4263304005486683740/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T16:30:36,076 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@776c4d96{HTTP/1.1, (http/1.1)}{localhost:43967} 2024-12-09T16:30:36,076 INFO [Time-limited test {}] server.Server(415): Started @3127ms 2024-12-09T16:30:36,648 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T16:30:36,661 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T16:30:36,673 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T16:30:36,673 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T16:30:36,673 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-09T16:30:36,674 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b895d95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/hadoop.log.dir/,AVAILABLE} 2024-12-09T16:30:36,675 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3d33512e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T16:30:36,782 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4dc44ec6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/java.io.tmpdir/jetty-localhost-33533-hadoop-hdfs-3_4_1-tests_jar-_-any-13351901834223837877/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T16:30:36,783 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@17f6e478{HTTP/1.1, (http/1.1)}{localhost:33533} 2024-12-09T16:30:36,784 INFO [Time-limited test {}] server.Server(415): Started @3834ms 2024-12-09T16:30:36,843 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T16:30:36,978 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T16:30:36,985 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T16:30:36,986 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T16:30:36,986 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T16:30:36,986 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T16:30:36,987 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e971547{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/hadoop.log.dir/,AVAILABLE} 2024-12-09T16:30:36,988 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3264f7d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T16:30:37,108 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@28e1ba78{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/java.io.tmpdir/jetty-localhost-41405-hadoop-hdfs-3_4_1-tests_jar-_-any-7742496352450448194/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T16:30:37,108 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@702c0733{HTTP/1.1, (http/1.1)}{localhost:41405} 2024-12-09T16:30:37,109 INFO [Time-limited test {}] server.Server(415): Started @4159ms 2024-12-09T16:30:37,111 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T16:30:37,151 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T16:30:37,155 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T16:30:37,156 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T16:30:37,156 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T16:30:37,156 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T16:30:37,157 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bf0fdb4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/hadoop.log.dir/,AVAILABLE} 2024-12-09T16:30:37,157 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6853f5bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T16:30:37,251 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4290616c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/java.io.tmpdir/jetty-localhost-42545-hadoop-hdfs-3_4_1-tests_jar-_-any-14520235980283518394/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T16:30:37,251 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5e25d2b6{HTTP/1.1, (http/1.1)}{localhost:42545} 2024-12-09T16:30:37,252 INFO [Time-limited test {}] server.Server(415): Started @4302ms 2024-12-09T16:30:37,254 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T16:30:38,844 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/cluster_cd98e670-d246-b6f3-ff89-5ca1891baf9f/data/data1/current/BP-427817209-172.17.0.2-1733761835089/current, will proceed with Du for space computation calculation, 2024-12-09T16:30:38,844 WARN [Thread-129 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/cluster_cd98e670-d246-b6f3-ff89-5ca1891baf9f/data/data6/current/BP-427817209-172.17.0.2-1733761835089/current, will proceed with Du for space computation calculation, 2024-12-09T16:30:38,844 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/cluster_cd98e670-d246-b6f3-ff89-5ca1891baf9f/data/data2/current/BP-427817209-172.17.0.2-1733761835089/current, will proceed with Du for space computation calculation, 2024-12-09T16:30:38,844 WARN [Thread-128 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/cluster_cd98e670-d246-b6f3-ff89-5ca1891baf9f/data/data5/current/BP-427817209-172.17.0.2-1733761835089/current, will proceed with Du for space computation calculation, 2024-12-09T16:30:38,844 WARN [Thread-126 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/cluster_cd98e670-d246-b6f3-ff89-5ca1891baf9f/data/data4/current/BP-427817209-172.17.0.2-1733761835089/current, will proceed with Du for space computation calculation, 2024-12-09T16:30:38,844 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/cluster_cd98e670-d246-b6f3-ff89-5ca1891baf9f/data/data3/current/BP-427817209-172.17.0.2-1733761835089/current, will proceed with Du for space computation calculation, 2024-12-09T16:30:38,893 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T16:30:38,893 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T16:30:38,893 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T16:30:38,951 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6346de75e13adb1d with lease ID 0x318921d21e4a0d4c: Processing first storage report for DS-36c2801e-54a9-44e5-a948-0b1b1a47424d from datanode DatanodeRegistration(127.0.0.1:36895, datanodeUuid=f89b2c38-2a6a-48a7-a790-f1f498b2e863, infoPort=35519, infoSecurePort=0, ipcPort=41135, storageInfo=lv=-57;cid=testClusterID;nsid=22398437;c=1733761835089) 2024-12-09T16:30:38,952 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6346de75e13adb1d with lease ID 0x318921d21e4a0d4c: from storage DS-36c2801e-54a9-44e5-a948-0b1b1a47424d node DatanodeRegistration(127.0.0.1:36895, datanodeUuid=f89b2c38-2a6a-48a7-a790-f1f498b2e863, infoPort=35519, infoSecurePort=0, ipcPort=41135, storageInfo=lv=-57;cid=testClusterID;nsid=22398437;c=1733761835089), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-09T16:30:38,953 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe39ac66cf6561d7d with lease ID 0x318921d21e4a0d4d: Processing first storage report for DS-b118a02d-bd6c-42db-be6e-e730304ac329 from datanode DatanodeRegistration(127.0.0.1:41451, datanodeUuid=1ef52798-38bb-448b-afff-499d10e83cc1, infoPort=37099, infoSecurePort=0, ipcPort=36531, storageInfo=lv=-57;cid=testClusterID;nsid=22398437;c=1733761835089) 2024-12-09T16:30:38,953 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe39ac66cf6561d7d with lease ID 0x318921d21e4a0d4d: from storage DS-b118a02d-bd6c-42db-be6e-e730304ac329 node DatanodeRegistration(127.0.0.1:41451, datanodeUuid=1ef52798-38bb-448b-afff-499d10e83cc1, infoPort=37099, infoSecurePort=0, ipcPort=36531, storageInfo=lv=-57;cid=testClusterID;nsid=22398437;c=1733761835089), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T16:30:38,953 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe40ba1c16b087ba0 with lease ID 0x318921d21e4a0d4e: Processing first storage report for DS-988cafaf-517a-41ae-b218-7e4ffd5ee141 from datanode DatanodeRegistration(127.0.0.1:39721, datanodeUuid=54b3487c-0a7e-4cdd-b31d-17fa0c40dd20, infoPort=39661, infoSecurePort=0, ipcPort=40409, storageInfo=lv=-57;cid=testClusterID;nsid=22398437;c=1733761835089) 2024-12-09T16:30:38,953 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe40ba1c16b087ba0 with lease ID 0x318921d21e4a0d4e: from storage DS-988cafaf-517a-41ae-b218-7e4ffd5ee141 node DatanodeRegistration(127.0.0.1:39721, datanodeUuid=54b3487c-0a7e-4cdd-b31d-17fa0c40dd20, infoPort=39661, infoSecurePort=0, ipcPort=40409, storageInfo=lv=-57;cid=testClusterID;nsid=22398437;c=1733761835089), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T16:30:38,954 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6346de75e13adb1d with lease ID 0x318921d21e4a0d4c: Processing first storage report for DS-c540f16a-ea29-4fcd-8855-42ab8ded4792 from datanode DatanodeRegistration(127.0.0.1:36895, datanodeUuid=f89b2c38-2a6a-48a7-a790-f1f498b2e863, infoPort=35519, infoSecurePort=0, ipcPort=41135, storageInfo=lv=-57;cid=testClusterID;nsid=22398437;c=1733761835089) 2024-12-09T16:30:38,954 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6346de75e13adb1d with lease ID 0x318921d21e4a0d4c: from storage DS-c540f16a-ea29-4fcd-8855-42ab8ded4792 node DatanodeRegistration(127.0.0.1:36895, datanodeUuid=f89b2c38-2a6a-48a7-a790-f1f498b2e863, infoPort=35519, infoSecurePort=0, ipcPort=41135, storageInfo=lv=-57;cid=testClusterID;nsid=22398437;c=1733761835089), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T16:30:38,954 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe39ac66cf6561d7d with lease ID 0x318921d21e4a0d4d: Processing first storage report for DS-1ad36fc0-c607-4681-bd9d-e4b4bbe48d8f from datanode DatanodeRegistration(127.0.0.1:41451, datanodeUuid=1ef52798-38bb-448b-afff-499d10e83cc1, infoPort=37099, infoSecurePort=0, ipcPort=36531, storageInfo=lv=-57;cid=testClusterID;nsid=22398437;c=1733761835089) 2024-12-09T16:30:38,954 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe39ac66cf6561d7d with lease ID 0x318921d21e4a0d4d: from storage DS-1ad36fc0-c607-4681-bd9d-e4b4bbe48d8f node DatanodeRegistration(127.0.0.1:41451, datanodeUuid=1ef52798-38bb-448b-afff-499d10e83cc1, infoPort=37099, infoSecurePort=0, ipcPort=36531, storageInfo=lv=-57;cid=testClusterID;nsid=22398437;c=1733761835089), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T16:30:38,954 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe40ba1c16b087ba0 with lease ID 0x318921d21e4a0d4e: Processing first storage report for DS-eb5db369-2a23-4dcd-98a7-084ff84e5411 from datanode DatanodeRegistration(127.0.0.1:39721, datanodeUuid=54b3487c-0a7e-4cdd-b31d-17fa0c40dd20, infoPort=39661, infoSecurePort=0, ipcPort=40409, storageInfo=lv=-57;cid=testClusterID;nsid=22398437;c=1733761835089) 2024-12-09T16:30:38,954 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe40ba1c16b087ba0 with lease ID 0x318921d21e4a0d4e: from storage DS-eb5db369-2a23-4dcd-98a7-084ff84e5411 node DatanodeRegistration(127.0.0.1:39721, datanodeUuid=54b3487c-0a7e-4cdd-b31d-17fa0c40dd20, infoPort=39661, infoSecurePort=0, ipcPort=40409, storageInfo=lv=-57;cid=testClusterID;nsid=22398437;c=1733761835089), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T16:30:38,996 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52 2024-12-09T16:30:39,097 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/cluster_cd98e670-d246-b6f3-ff89-5ca1891baf9f/zookeeper_0, clientPort=60962, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/cluster_cd98e670-d246-b6f3-ff89-5ca1891baf9f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/cluster_cd98e670-d246-b6f3-ff89-5ca1891baf9f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T16:30:39,124 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60962 2024-12-09T16:30:39,142 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T16:30:39,146 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T16:30:39,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741825_1001 (size=7) 2024-12-09T16:30:39,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741825_1001 (size=7) 2024-12-09T16:30:39,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741825_1001 (size=7) 2024-12-09T16:30:39,768 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38 with version=8 2024-12-09T16:30:39,769 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/hbase-staging 2024-12-09T16:30:40,037 INFO [Time-limited test {}] client.ConnectionUtils(128): master/dfce6ce5b10c:0 server-side Connection retries=45 2024-12-09T16:30:40,046 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T16:30:40,046 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T16:30:40,051 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T16:30:40,051 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T16:30:40,051 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T16:30:40,179 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-09T16:30:40,237 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T16:30:40,245 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T16:30:40,249 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T16:30:40,272 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 27853 (auto-detected) 2024-12-09T16:30:40,273 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-09T16:30:40,291 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35827 2024-12-09T16:30:40,311 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35827 connecting to ZooKeeper ensemble=127.0.0.1:60962 2024-12-09T16:30:40,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:358270x0, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T16:30:40,411 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35827-0x100bfc42e970000 connected 2024-12-09T16:30:40,487 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T16:30:40,490 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T16:30:40,501 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T16:30:40,506 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38, hbase.cluster.distributed=false 2024-12-09T16:30:40,532 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T16:30:40,538 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35827 2024-12-09T16:30:40,538 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35827 2024-12-09T16:30:40,539 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35827 2024-12-09T16:30:40,545 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35827 2024-12-09T16:30:40,545 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35827 2024-12-09T16:30:40,687 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/dfce6ce5b10c:0 server-side Connection retries=45 2024-12-09T16:30:40,689 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T16:30:40,690 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T16:30:40,690 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T16:30:40,690 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T16:30:40,691 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T16:30:40,694 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T16:30:40,697 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T16:30:40,698 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38861 2024-12-09T16:30:40,700 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38861 connecting to ZooKeeper ensemble=127.0.0.1:60962 2024-12-09T16:30:40,702 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T16:30:40,707 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T16:30:40,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:388610x0, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T16:30:40,727 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38861-0x100bfc42e970001 connected 2024-12-09T16:30:40,728 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38861-0x100bfc42e970001, quorum=127.0.0.1:60962, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T16:30:40,733 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T16:30:40,749 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T16:30:40,752 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38861-0x100bfc42e970001, quorum=127.0.0.1:60962, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T16:30:40,759 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38861-0x100bfc42e970001, quorum=127.0.0.1:60962, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T16:30:40,763 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38861 2024-12-09T16:30:40,765 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38861 2024-12-09T16:30:40,770 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38861 2024-12-09T16:30:40,771 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38861 2024-12-09T16:30:40,772 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38861 2024-12-09T16:30:40,799 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/dfce6ce5b10c:0 server-side Connection retries=45 2024-12-09T16:30:40,799 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T16:30:40,799 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T16:30:40,800 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T16:30:40,800 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T16:30:40,800 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T16:30:40,801 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T16:30:40,801 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T16:30:40,802 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32855 2024-12-09T16:30:40,805 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32855 connecting to ZooKeeper ensemble=127.0.0.1:60962 2024-12-09T16:30:40,806 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T16:30:40,810 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T16:30:40,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:328550x0, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T16:30:40,827 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32855-0x100bfc42e970002 connected 2024-12-09T16:30:40,827 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:328550x0, quorum=127.0.0.1:60962, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T16:30:40,828 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T16:30:40,829 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T16:30:40,830 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32855-0x100bfc42e970002, quorum=127.0.0.1:60962, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T16:30:40,833 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32855-0x100bfc42e970002, quorum=127.0.0.1:60962, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T16:30:40,838 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32855 2024-12-09T16:30:40,838 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32855 2024-12-09T16:30:40,839 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32855 2024-12-09T16:30:40,843 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32855 2024-12-09T16:30:40,844 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32855 2024-12-09T16:30:40,862 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/dfce6ce5b10c:0 server-side Connection retries=45 2024-12-09T16:30:40,862 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T16:30:40,862 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T16:30:40,863 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T16:30:40,863 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T16:30:40,863 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T16:30:40,863 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T16:30:40,863 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T16:30:40,864 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34709 2024-12-09T16:30:40,867 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34709 connecting to ZooKeeper ensemble=127.0.0.1:60962 2024-12-09T16:30:40,869 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T16:30:40,873 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T16:30:40,885 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:347090x0, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T16:30:40,886 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:347090x0, quorum=127.0.0.1:60962, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T16:30:40,886 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34709-0x100bfc42e970003 connected 2024-12-09T16:30:40,887 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T16:30:40,888 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T16:30:40,890 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34709-0x100bfc42e970003, quorum=127.0.0.1:60962, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T16:30:40,893 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34709-0x100bfc42e970003, quorum=127.0.0.1:60962, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T16:30:40,894 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34709 2024-12-09T16:30:40,898 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34709 2024-12-09T16:30:40,900 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34709 2024-12-09T16:30:40,901 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34709 2024-12-09T16:30:40,902 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34709 2024-12-09T16:30:40,917 DEBUG [M:0;dfce6ce5b10c:35827 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;dfce6ce5b10c:35827 2024-12-09T16:30:40,919 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/dfce6ce5b10c,35827,1733761839880 2024-12-09T16:30:40,935 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34709-0x100bfc42e970003, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T16:30:40,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T16:30:40,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38861-0x100bfc42e970001, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T16:30:40,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32855-0x100bfc42e970002, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T16:30:40,938 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/dfce6ce5b10c,35827,1733761839880 2024-12-09T16:30:40,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32855-0x100bfc42e970002, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T16:30:40,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38861-0x100bfc42e970001, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T16:30:40,960 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34709-0x100bfc42e970003, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T16:30:40,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32855-0x100bfc42e970002, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:30:40,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38861-0x100bfc42e970001, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:30:40,960 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34709-0x100bfc42e970003, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:30:40,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:30:40,962 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T16:30:40,963 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/dfce6ce5b10c,35827,1733761839880 from backup master directory 2024-12-09T16:30:40,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/dfce6ce5b10c,35827,1733761839880 2024-12-09T16:30:40,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38861-0x100bfc42e970001, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T16:30:40,968 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34709-0x100bfc42e970003, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T16:30:40,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32855-0x100bfc42e970002, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T16:30:40,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T16:30:40,970 WARN [master/dfce6ce5b10c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T16:30:40,970 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=dfce6ce5b10c,35827,1733761839880 2024-12-09T16:30:40,972 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-09T16:30:40,974 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-09T16:30:41,031 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/hbase.id] with ID: 80e51bf9-2430-42cd-8179-eacd4b0999ae 2024-12-09T16:30:41,031 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/.tmp/hbase.id 2024-12-09T16:30:41,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741826_1002 (size=42) 2024-12-09T16:30:41,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741826_1002 (size=42) 2024-12-09T16:30:41,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741826_1002 (size=42) 2024-12-09T16:30:41,049 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/.tmp/hbase.id]:[hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/hbase.id] 2024-12-09T16:30:41,103 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T16:30:41,109 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-09T16:30:41,130 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-12-09T16:30:41,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38861-0x100bfc42e970001, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:30:41,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:30:41,143 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34709-0x100bfc42e970003, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:30:41,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32855-0x100bfc42e970002, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:30:41,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741827_1003 (size=196) 2024-12-09T16:30:41,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741827_1003 (size=196) 2024-12-09T16:30:41,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741827_1003 (size=196) 2024-12-09T16:30:41,181 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T16:30:41,183 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T16:30:41,190 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:30:41,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741828_1004 (size=1189) 2024-12-09T16:30:41,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741828_1004 (size=1189) 2024-12-09T16:30:41,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741828_1004 (size=1189) 2024-12-09T16:30:41,245 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/MasterData/data/master/store 2024-12-09T16:30:41,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741829_1005 (size=34) 2024-12-09T16:30:41,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741829_1005 (size=34) 2024-12-09T16:30:41,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741829_1005 (size=34) 2024-12-09T16:30:41,274 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T16:30:41,277 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:30:41,279 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T16:30:41,280 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T16:30:41,280 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T16:30:41,282 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T16:30:41,282 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T16:30:41,282 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T16:30:41,283 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733761841279Disabling compacts and flushes for region at 1733761841279Disabling writes for close at 1733761841282 (+3 ms)Writing region close event to WAL at 1733761841282Closed at 1733761841282 2024-12-09T16:30:41,286 WARN [master/dfce6ce5b10c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/MasterData/data/master/store/.initializing 2024-12-09T16:30:41,286 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/MasterData/WALs/dfce6ce5b10c,35827,1733761839880 2024-12-09T16:30:41,296 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T16:30:41,313 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dfce6ce5b10c%2C35827%2C1733761839880, suffix=, logDir=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/MasterData/WALs/dfce6ce5b10c,35827,1733761839880, archiveDir=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/MasterData/oldWALs, maxLogs=10 2024-12-09T16:30:41,366 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/MasterData/WALs/dfce6ce5b10c,35827,1733761839880/dfce6ce5b10c%2C35827%2C1733761839880.1733761841320, exclude list is [], retry=0 2024-12-09T16:30:41,388 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:30:41,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:30:41,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:30:41,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:30:41,399 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-09T16:30:41,444 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/MasterData/WALs/dfce6ce5b10c,35827,1733761839880/dfce6ce5b10c%2C35827%2C1733761839880.1733761841320 2024-12-09T16:30:41,445 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:39661:39661),(127.0.0.1/127.0.0.1:37099:37099)] 2024-12-09T16:30:41,446 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T16:30:41,446 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:30:41,449 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T16:30:41,450 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T16:30:41,489 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T16:30:41,517 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T16:30:41,521 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:41,524 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T16:30:41,525 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T16:30:41,529 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T16:30:41,529 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:41,531 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:41,531 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T16:30:41,534 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T16:30:41,534 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:41,536 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:41,536 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T16:30:41,539 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T16:30:41,540 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:41,541 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:41,541 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T16:30:41,545 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T16:30:41,547 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T16:30:41,552 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T16:30:41,552 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T16:30:41,556 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T16:30:41,559 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T16:30:41,564 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T16:30:41,566 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64883484, jitterRate=-0.0331607460975647}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T16:30:41,571 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733761841462Initializing all the Stores at 1733761841464 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733761841465 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761841465Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761841466 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761841466Cleaning up temporary data from old regions at 1733761841552 (+86 ms)Region opened successfully at 1733761841571 (+19 ms) 2024-12-09T16:30:41,573 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T16:30:41,608 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17a617c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dfce6ce5b10c/172.17.0.2:0 2024-12-09T16:30:41,638 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-09T16:30:41,649 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-09T16:30:41,649 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-09T16:30:41,653 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T16:30:41,654 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-09T16:30:41,659 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-09T16:30:41,659 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T16:30:41,686 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T16:30:41,694 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T16:30:41,734 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-09T16:30:41,738 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T16:30:41,740 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T16:30:41,751 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-09T16:30:41,755 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T16:30:41,760 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T16:30:41,768 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-09T16:30:41,769 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T16:30:41,776 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T16:30:41,793 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T16:30:41,801 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T16:30:41,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T16:30:41,810 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34709-0x100bfc42e970003, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T16:30:41,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:30:41,810 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34709-0x100bfc42e970003, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:30:41,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38861-0x100bfc42e970001, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T16:30:41,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32855-0x100bfc42e970002, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T16:30:41,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38861-0x100bfc42e970001, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:30:41,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32855-0x100bfc42e970002, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:30:41,813 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=dfce6ce5b10c,35827,1733761839880, sessionid=0x100bfc42e970000, setting cluster-up flag (Was=false) 2024-12-09T16:30:41,843 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34709-0x100bfc42e970003, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:30:41,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:30:41,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32855-0x100bfc42e970002, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:30:41,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38861-0x100bfc42e970001, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:30:41,868 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T16:30:41,870 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dfce6ce5b10c,35827,1733761839880 2024-12-09T16:30:41,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38861-0x100bfc42e970001, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:30:41,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:30:41,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32855-0x100bfc42e970002, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:30:41,884 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34709-0x100bfc42e970003, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:30:41,909 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T16:30:41,911 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dfce6ce5b10c,35827,1733761839880 2024-12-09T16:30:41,919 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-09T16:30:42,006 INFO [RS:2;dfce6ce5b10c:34709 {}] regionserver.HRegionServer(746): ClusterId : 80e51bf9-2430-42cd-8179-eacd4b0999ae 2024-12-09T16:30:42,006 INFO [RS:0;dfce6ce5b10c:38861 {}] regionserver.HRegionServer(746): ClusterId : 80e51bf9-2430-42cd-8179-eacd4b0999ae 2024-12-09T16:30:42,006 INFO [RS:1;dfce6ce5b10c:32855 {}] regionserver.HRegionServer(746): ClusterId : 80e51bf9-2430-42cd-8179-eacd4b0999ae 2024-12-09T16:30:42,009 DEBUG [RS:1;dfce6ce5b10c:32855 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T16:30:42,009 DEBUG [RS:0;dfce6ce5b10c:38861 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T16:30:42,009 DEBUG [RS:2;dfce6ce5b10c:34709 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T16:30:42,014 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T16:30:42,025 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-09T16:30:42,028 DEBUG [RS:0;dfce6ce5b10c:38861 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T16:30:42,028 DEBUG [RS:2;dfce6ce5b10c:34709 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T16:30:42,028 DEBUG [RS:1;dfce6ce5b10c:32855 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T16:30:42,028 DEBUG [RS:0;dfce6ce5b10c:38861 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T16:30:42,028 DEBUG [RS:1;dfce6ce5b10c:32855 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T16:30:42,028 DEBUG [RS:2;dfce6ce5b10c:34709 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T16:30:42,033 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T16:30:42,040 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: dfce6ce5b10c,35827,1733761839880 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T16:30:42,044 DEBUG [RS:0;dfce6ce5b10c:38861 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T16:30:42,044 DEBUG [RS:2;dfce6ce5b10c:34709 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T16:30:42,044 DEBUG [RS:1;dfce6ce5b10c:32855 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T16:30:42,044 DEBUG [RS:0;dfce6ce5b10c:38861 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e256a67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dfce6ce5b10c/172.17.0.2:0 2024-12-09T16:30:42,044 DEBUG [RS:1;dfce6ce5b10c:32855 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b388fe4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dfce6ce5b10c/172.17.0.2:0 2024-12-09T16:30:42,044 DEBUG [RS:2;dfce6ce5b10c:34709 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4968f7de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dfce6ce5b10c/172.17.0.2:0 2024-12-09T16:30:42,048 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/dfce6ce5b10c:0, corePoolSize=5, maxPoolSize=5 2024-12-09T16:30:42,048 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/dfce6ce5b10c:0, corePoolSize=5, maxPoolSize=5 2024-12-09T16:30:42,049 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/dfce6ce5b10c:0, corePoolSize=5, maxPoolSize=5 2024-12-09T16:30:42,049 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/dfce6ce5b10c:0, corePoolSize=5, maxPoolSize=5 2024-12-09T16:30:42,049 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/dfce6ce5b10c:0, corePoolSize=10, maxPoolSize=10 2024-12-09T16:30:42,049 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,049 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/dfce6ce5b10c:0, corePoolSize=2, maxPoolSize=2 2024-12-09T16:30:42,049 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,062 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T16:30:42,062 DEBUG [RS:2;dfce6ce5b10c:34709 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;dfce6ce5b10c:34709 2024-12-09T16:30:42,063 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-09T16:30:42,064 DEBUG [RS:1;dfce6ce5b10c:32855 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;dfce6ce5b10c:32855 2024-12-09T16:30:42,065 DEBUG [RS:0;dfce6ce5b10c:38861 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;dfce6ce5b10c:38861 2024-12-09T16:30:42,067 INFO [RS:0;dfce6ce5b10c:38861 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T16:30:42,067 INFO [RS:1;dfce6ce5b10c:32855 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T16:30:42,067 INFO [RS:2;dfce6ce5b10c:34709 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T16:30:42,067 INFO [RS:0;dfce6ce5b10c:38861 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T16:30:42,067 INFO [RS:1;dfce6ce5b10c:32855 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T16:30:42,067 INFO [RS:2;dfce6ce5b10c:34709 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T16:30:42,067 DEBUG [RS:0;dfce6ce5b10c:38861 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T16:30:42,067 DEBUG [RS:1;dfce6ce5b10c:32855 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T16:30:42,067 DEBUG [RS:2;dfce6ce5b10c:34709 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T16:30:42,075 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733761872074 2024-12-09T16:30:42,075 INFO [RS:1;dfce6ce5b10c:32855 {}] regionserver.HRegionServer(2659): reportForDuty to master=dfce6ce5b10c,35827,1733761839880 with port=32855, startcode=1733761840798 2024-12-09T16:30:42,075 INFO [RS:0;dfce6ce5b10c:38861 {}] regionserver.HRegionServer(2659): reportForDuty to master=dfce6ce5b10c,35827,1733761839880 with port=38861, startcode=1733761840646 2024-12-09T16:30:42,076 INFO [RS:2;dfce6ce5b10c:34709 {}] regionserver.HRegionServer(2659): reportForDuty to master=dfce6ce5b10c,35827,1733761839880 with port=34709, startcode=1733761840861 2024-12-09T16:30:42,076 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:42,076 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T16:30:42,077 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T16:30:42,078 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T16:30:42,082 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T16:30:42,082 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T16:30:42,082 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T16:30:42,082 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T16:30:42,089 DEBUG [RS:2;dfce6ce5b10c:34709 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T16:30:42,089 DEBUG [RS:0;dfce6ce5b10c:38861 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T16:30:42,090 DEBUG [RS:1;dfce6ce5b10c:32855 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T16:30:42,088 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,093 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T16:30:42,094 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T16:30:42,095 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T16:30:42,099 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T16:30:42,099 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T16:30:42,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741831_1007 (size=1321) 2024-12-09T16:30:42,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741831_1007 (size=1321) 2024-12-09T16:30:42,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741831_1007 (size=1321) 2024-12-09T16:30:42,108 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-09T16:30:42,108 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38 2024-12-09T16:30:42,110 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/dfce6ce5b10c:0:becomeActiveMaster-HFileCleaner.large.0-1733761842101,5,FailOnTimeoutGroup] 2024-12-09T16:30:42,114 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/dfce6ce5b10c:0:becomeActiveMaster-HFileCleaner.small.0-1733761842110,5,FailOnTimeoutGroup] 2024-12-09T16:30:42,114 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,114 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T16:30:42,115 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,115 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741832_1008 (size=32) 2024-12-09T16:30:42,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741832_1008 (size=32) 2024-12-09T16:30:42,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741832_1008 (size=32) 2024-12-09T16:30:42,132 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:30:42,138 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T16:30:42,141 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T16:30:42,142 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:42,143 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T16:30:42,143 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T16:30:42,146 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T16:30:42,147 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:42,150 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T16:30:42,151 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T16:30:42,155 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T16:30:42,155 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:42,163 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T16:30:42,163 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T16:30:42,169 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T16:30:42,169 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:42,171 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T16:30:42,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T16:30:42,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/hbase/meta/1588230740 2024-12-09T16:30:42,174 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/hbase/meta/1588230740 2024-12-09T16:30:42,177 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49461, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T16:30:42,178 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T16:30:42,178 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T16:30:42,178 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46895, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T16:30:42,179 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47855, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T16:30:42,179 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T16:30:42,183 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T16:30:42,186 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35827 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dfce6ce5b10c,32855,1733761840798 2024-12-09T16:30:42,190 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35827 {}] master.ServerManager(517): Registering regionserver=dfce6ce5b10c,32855,1733761840798 2024-12-09T16:30:42,191 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T16:30:42,193 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60881314, jitterRate=-0.09279772639274597}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T16:30:42,198 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733761842132Initializing all the Stores at 1733761842136 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733761842136Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733761842138 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761842138Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733761842138Cleaning up temporary data from old regions at 1733761842178 (+40 ms)Region opened successfully at 1733761842198 (+20 ms) 2024-12-09T16:30:42,198 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T16:30:42,198 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T16:30:42,198 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T16:30:42,199 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T16:30:42,199 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T16:30:42,202 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T16:30:42,203 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733761842198Disabling compacts and flushes for region at 1733761842198Disabling writes for close at 1733761842199 (+1 ms)Writing region close event to WAL at 1733761842202 (+3 ms)Closed at 1733761842202 2024-12-09T16:30:42,207 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35827 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dfce6ce5b10c,38861,1733761840646 2024-12-09T16:30:42,207 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35827 {}] master.ServerManager(517): Registering regionserver=dfce6ce5b10c,38861,1733761840646 2024-12-09T16:30:42,208 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T16:30:42,208 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-09T16:30:42,211 DEBUG [RS:1;dfce6ce5b10c:32855 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38 2024-12-09T16:30:42,211 DEBUG [RS:1;dfce6ce5b10c:32855 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39607 2024-12-09T16:30:42,212 DEBUG [RS:1;dfce6ce5b10c:32855 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T16:30:42,214 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35827 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dfce6ce5b10c,34709,1733761840861 2024-12-09T16:30:42,215 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35827 {}] master.ServerManager(517): Registering regionserver=dfce6ce5b10c,34709,1733761840861 2024-12-09T16:30:42,215 DEBUG [RS:0;dfce6ce5b10c:38861 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38 2024-12-09T16:30:42,215 DEBUG [RS:0;dfce6ce5b10c:38861 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39607 2024-12-09T16:30:42,215 DEBUG [RS:0;dfce6ce5b10c:38861 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T16:30:42,219 DEBUG [RS:2;dfce6ce5b10c:34709 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38 2024-12-09T16:30:42,219 DEBUG [RS:2;dfce6ce5b10c:34709 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39607 2024-12-09T16:30:42,219 DEBUG [RS:2;dfce6ce5b10c:34709 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T16:30:42,222 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T16:30:42,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T16:30:42,233 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T16:30:42,236 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T16:30:42,279 DEBUG [RS:1;dfce6ce5b10c:32855 {}] zookeeper.ZKUtil(111): regionserver:32855-0x100bfc42e970002, quorum=127.0.0.1:60962, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dfce6ce5b10c,32855,1733761840798 2024-12-09T16:30:42,280 WARN [RS:1;dfce6ce5b10c:32855 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T16:30:42,280 DEBUG [RS:2;dfce6ce5b10c:34709 {}] zookeeper.ZKUtil(111): regionserver:34709-0x100bfc42e970003, quorum=127.0.0.1:60962, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dfce6ce5b10c,34709,1733761840861 2024-12-09T16:30:42,280 INFO [RS:1;dfce6ce5b10c:32855 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:30:42,280 DEBUG [RS:0;dfce6ce5b10c:38861 {}] zookeeper.ZKUtil(111): regionserver:38861-0x100bfc42e970001, quorum=127.0.0.1:60962, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dfce6ce5b10c,38861,1733761840646 2024-12-09T16:30:42,280 WARN [RS:2;dfce6ce5b10c:34709 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T16:30:42,280 WARN [RS:0;dfce6ce5b10c:38861 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T16:30:42,280 INFO [RS:2;dfce6ce5b10c:34709 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:30:42,280 DEBUG [RS:1;dfce6ce5b10c:32855 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,32855,1733761840798 2024-12-09T16:30:42,280 INFO [RS:0;dfce6ce5b10c:38861 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:30:42,280 DEBUG [RS:2;dfce6ce5b10c:34709 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,34709,1733761840861 2024-12-09T16:30:42,280 DEBUG [RS:0;dfce6ce5b10c:38861 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,38861,1733761840646 2024-12-09T16:30:42,282 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dfce6ce5b10c,32855,1733761840798] 2024-12-09T16:30:42,282 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dfce6ce5b10c,34709,1733761840861] 2024-12-09T16:30:42,282 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dfce6ce5b10c,38861,1733761840646] 2024-12-09T16:30:42,315 INFO [RS:2;dfce6ce5b10c:34709 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T16:30:42,315 INFO [RS:0;dfce6ce5b10c:38861 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T16:30:42,315 INFO [RS:1;dfce6ce5b10c:32855 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T16:30:42,330 INFO [RS:1;dfce6ce5b10c:32855 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T16:30:42,330 INFO [RS:2;dfce6ce5b10c:34709 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T16:30:42,330 INFO [RS:0;dfce6ce5b10c:38861 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T16:30:42,338 INFO [RS:1;dfce6ce5b10c:32855 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T16:30:42,338 INFO [RS:0;dfce6ce5b10c:38861 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T16:30:42,338 INFO [RS:2;dfce6ce5b10c:34709 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T16:30:42,338 INFO [RS:1;dfce6ce5b10c:32855 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,338 INFO [RS:2;dfce6ce5b10c:34709 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,338 INFO [RS:0;dfce6ce5b10c:38861 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,339 INFO [RS:0;dfce6ce5b10c:38861 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T16:30:42,339 INFO [RS:1;dfce6ce5b10c:32855 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T16:30:42,340 INFO [RS:2;dfce6ce5b10c:34709 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T16:30:42,346 INFO [RS:2;dfce6ce5b10c:34709 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T16:30:42,346 INFO [RS:0;dfce6ce5b10c:38861 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T16:30:42,346 INFO [RS:1;dfce6ce5b10c:32855 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T16:30:42,348 INFO [RS:2;dfce6ce5b10c:34709 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,348 INFO [RS:0;dfce6ce5b10c:38861 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,348 INFO [RS:1;dfce6ce5b10c:32855 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,348 DEBUG [RS:0;dfce6ce5b10c:38861 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,348 DEBUG [RS:2;dfce6ce5b10c:34709 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,348 DEBUG [RS:1;dfce6ce5b10c:32855 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,349 DEBUG [RS:2;dfce6ce5b10c:34709 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,349 DEBUG [RS:0;dfce6ce5b10c:38861 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,349 DEBUG [RS:1;dfce6ce5b10c:32855 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,349 DEBUG [RS:1;dfce6ce5b10c:32855 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,349 DEBUG [RS:0;dfce6ce5b10c:38861 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,349 DEBUG [RS:2;dfce6ce5b10c:34709 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,349 DEBUG [RS:0;dfce6ce5b10c:38861 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,349 DEBUG [RS:2;dfce6ce5b10c:34709 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,349 DEBUG [RS:1;dfce6ce5b10c:32855 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,350 DEBUG [RS:0;dfce6ce5b10c:38861 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,350 DEBUG [RS:2;dfce6ce5b10c:34709 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,350 DEBUG [RS:1;dfce6ce5b10c:32855 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,350 DEBUG [RS:0;dfce6ce5b10c:38861 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dfce6ce5b10c:0, corePoolSize=2, maxPoolSize=2 2024-12-09T16:30:42,350 DEBUG [RS:2;dfce6ce5b10c:34709 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dfce6ce5b10c:0, corePoolSize=2, maxPoolSize=2 2024-12-09T16:30:42,350 DEBUG [RS:1;dfce6ce5b10c:32855 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dfce6ce5b10c:0, corePoolSize=2, maxPoolSize=2 2024-12-09T16:30:42,350 DEBUG [RS:0;dfce6ce5b10c:38861 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,350 DEBUG [RS:2;dfce6ce5b10c:34709 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,350 DEBUG [RS:1;dfce6ce5b10c:32855 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,350 DEBUG [RS:2;dfce6ce5b10c:34709 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,350 DEBUG [RS:1;dfce6ce5b10c:32855 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,350 DEBUG [RS:0;dfce6ce5b10c:38861 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,350 DEBUG [RS:2;dfce6ce5b10c:34709 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,350 DEBUG [RS:0;dfce6ce5b10c:38861 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,350 DEBUG [RS:1;dfce6ce5b10c:32855 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,350 DEBUG [RS:2;dfce6ce5b10c:34709 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,350 DEBUG [RS:0;dfce6ce5b10c:38861 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,350 DEBUG [RS:1;dfce6ce5b10c:32855 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,351 DEBUG [RS:2;dfce6ce5b10c:34709 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,351 DEBUG [RS:0;dfce6ce5b10c:38861 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,351 DEBUG [RS:1;dfce6ce5b10c:32855 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,351 DEBUG [RS:2;dfce6ce5b10c:34709 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,351 DEBUG [RS:0;dfce6ce5b10c:38861 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,351 DEBUG [RS:1;dfce6ce5b10c:32855 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dfce6ce5b10c:0, corePoolSize=1, maxPoolSize=1 2024-12-09T16:30:42,351 DEBUG [RS:2;dfce6ce5b10c:34709 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dfce6ce5b10c:0, corePoolSize=3, maxPoolSize=3 2024-12-09T16:30:42,351 DEBUG [RS:0;dfce6ce5b10c:38861 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dfce6ce5b10c:0, corePoolSize=3, maxPoolSize=3 2024-12-09T16:30:42,351 DEBUG [RS:1;dfce6ce5b10c:32855 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dfce6ce5b10c:0, corePoolSize=3, maxPoolSize=3 2024-12-09T16:30:42,351 DEBUG [RS:0;dfce6ce5b10c:38861 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dfce6ce5b10c:0, corePoolSize=3, maxPoolSize=3 2024-12-09T16:30:42,351 DEBUG [RS:2;dfce6ce5b10c:34709 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dfce6ce5b10c:0, corePoolSize=3, maxPoolSize=3 2024-12-09T16:30:42,351 DEBUG [RS:1;dfce6ce5b10c:32855 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dfce6ce5b10c:0, corePoolSize=3, maxPoolSize=3 2024-12-09T16:30:42,355 INFO [RS:0;dfce6ce5b10c:38861 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,355 INFO [RS:0;dfce6ce5b10c:38861 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,355 INFO [RS:0;dfce6ce5b10c:38861 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,356 INFO [RS:0;dfce6ce5b10c:38861 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,356 INFO [RS:0;dfce6ce5b10c:38861 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,356 INFO [RS:0;dfce6ce5b10c:38861 {}] hbase.ChoreService(168): Chore ScheduledChore name=dfce6ce5b10c,38861,1733761840646-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T16:30:42,358 INFO [RS:1;dfce6ce5b10c:32855 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,358 INFO [RS:1;dfce6ce5b10c:32855 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,358 INFO [RS:1;dfce6ce5b10c:32855 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,358 INFO [RS:1;dfce6ce5b10c:32855 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,358 INFO [RS:1;dfce6ce5b10c:32855 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,358 INFO [RS:1;dfce6ce5b10c:32855 {}] hbase.ChoreService(168): Chore ScheduledChore name=dfce6ce5b10c,32855,1733761840798-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T16:30:42,362 INFO [RS:2;dfce6ce5b10c:34709 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,362 INFO [RS:2;dfce6ce5b10c:34709 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,362 INFO [RS:2;dfce6ce5b10c:34709 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,363 INFO [RS:2;dfce6ce5b10c:34709 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,363 INFO [RS:2;dfce6ce5b10c:34709 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,363 INFO [RS:2;dfce6ce5b10c:34709 {}] hbase.ChoreService(168): Chore ScheduledChore name=dfce6ce5b10c,34709,1733761840861-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T16:30:42,384 INFO [RS:2;dfce6ce5b10c:34709 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T16:30:42,385 INFO [RS:0;dfce6ce5b10c:38861 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T16:30:42,385 INFO [RS:1;dfce6ce5b10c:32855 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T16:30:42,387 INFO [RS:2;dfce6ce5b10c:34709 {}] hbase.ChoreService(168): Chore ScheduledChore name=dfce6ce5b10c,34709,1733761840861-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,387 INFO [RS:0;dfce6ce5b10c:38861 {}] hbase.ChoreService(168): Chore ScheduledChore name=dfce6ce5b10c,38861,1733761840646-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,387 INFO [RS:0;dfce6ce5b10c:38861 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,387 INFO [RS:2;dfce6ce5b10c:34709 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,388 INFO [RS:0;dfce6ce5b10c:38861 {}] regionserver.Replication(171): dfce6ce5b10c,38861,1733761840646 started 2024-12-09T16:30:42,388 WARN [dfce6ce5b10c:35827 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T16:30:42,387 INFO [RS:1;dfce6ce5b10c:32855 {}] hbase.ChoreService(168): Chore ScheduledChore name=dfce6ce5b10c,32855,1733761840798-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,388 INFO [RS:2;dfce6ce5b10c:34709 {}] regionserver.Replication(171): dfce6ce5b10c,34709,1733761840861 started 2024-12-09T16:30:42,388 INFO [RS:1;dfce6ce5b10c:32855 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,388 INFO [RS:1;dfce6ce5b10c:32855 {}] regionserver.Replication(171): dfce6ce5b10c,32855,1733761840798 started 2024-12-09T16:30:42,413 INFO [RS:1;dfce6ce5b10c:32855 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,413 INFO [RS:0;dfce6ce5b10c:38861 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,413 INFO [RS:2;dfce6ce5b10c:34709 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:42,413 INFO [RS:1;dfce6ce5b10c:32855 {}] regionserver.HRegionServer(1482): Serving as dfce6ce5b10c,32855,1733761840798, RpcServer on dfce6ce5b10c/172.17.0.2:32855, sessionid=0x100bfc42e970002 2024-12-09T16:30:42,413 INFO [RS:0;dfce6ce5b10c:38861 {}] regionserver.HRegionServer(1482): Serving as dfce6ce5b10c,38861,1733761840646, RpcServer on dfce6ce5b10c/172.17.0.2:38861, sessionid=0x100bfc42e970001 2024-12-09T16:30:42,413 INFO [RS:2;dfce6ce5b10c:34709 {}] regionserver.HRegionServer(1482): Serving as dfce6ce5b10c,34709,1733761840861, RpcServer on dfce6ce5b10c/172.17.0.2:34709, sessionid=0x100bfc42e970003 2024-12-09T16:30:42,414 DEBUG [RS:1;dfce6ce5b10c:32855 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T16:30:42,414 DEBUG [RS:2;dfce6ce5b10c:34709 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T16:30:42,414 DEBUG [RS:0;dfce6ce5b10c:38861 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T16:30:42,414 DEBUG [RS:1;dfce6ce5b10c:32855 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dfce6ce5b10c,32855,1733761840798 2024-12-09T16:30:42,414 DEBUG [RS:2;dfce6ce5b10c:34709 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dfce6ce5b10c,34709,1733761840861 2024-12-09T16:30:42,414 DEBUG [RS:0;dfce6ce5b10c:38861 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dfce6ce5b10c,38861,1733761840646 2024-12-09T16:30:42,414 DEBUG [RS:1;dfce6ce5b10c:32855 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dfce6ce5b10c,32855,1733761840798' 2024-12-09T16:30:42,414 DEBUG [RS:2;dfce6ce5b10c:34709 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dfce6ce5b10c,34709,1733761840861' 2024-12-09T16:30:42,414 DEBUG [RS:0;dfce6ce5b10c:38861 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dfce6ce5b10c,38861,1733761840646' 2024-12-09T16:30:42,414 DEBUG [RS:1;dfce6ce5b10c:32855 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T16:30:42,414 DEBUG [RS:2;dfce6ce5b10c:34709 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T16:30:42,414 DEBUG [RS:0;dfce6ce5b10c:38861 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T16:30:42,416 DEBUG [RS:2;dfce6ce5b10c:34709 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T16:30:42,416 DEBUG [RS:0;dfce6ce5b10c:38861 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T16:30:42,416 DEBUG [RS:1;dfce6ce5b10c:32855 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T16:30:42,416 DEBUG [RS:0;dfce6ce5b10c:38861 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T16:30:42,416 DEBUG [RS:2;dfce6ce5b10c:34709 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T16:30:42,417 DEBUG [RS:0;dfce6ce5b10c:38861 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T16:30:42,417 DEBUG [RS:2;dfce6ce5b10c:34709 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T16:30:42,417 DEBUG [RS:1;dfce6ce5b10c:32855 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T16:30:42,417 DEBUG [RS:1;dfce6ce5b10c:32855 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T16:30:42,417 DEBUG [RS:0;dfce6ce5b10c:38861 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dfce6ce5b10c,38861,1733761840646 2024-12-09T16:30:42,417 DEBUG [RS:2;dfce6ce5b10c:34709 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dfce6ce5b10c,34709,1733761840861 2024-12-09T16:30:42,417 DEBUG [RS:1;dfce6ce5b10c:32855 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dfce6ce5b10c,32855,1733761840798 2024-12-09T16:30:42,417 DEBUG [RS:2;dfce6ce5b10c:34709 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dfce6ce5b10c,34709,1733761840861' 2024-12-09T16:30:42,417 DEBUG [RS:1;dfce6ce5b10c:32855 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dfce6ce5b10c,32855,1733761840798' 2024-12-09T16:30:42,417 DEBUG [RS:0;dfce6ce5b10c:38861 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dfce6ce5b10c,38861,1733761840646' 2024-12-09T16:30:42,417 DEBUG [RS:2;dfce6ce5b10c:34709 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T16:30:42,417 DEBUG [RS:1;dfce6ce5b10c:32855 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T16:30:42,417 DEBUG [RS:0;dfce6ce5b10c:38861 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T16:30:42,418 DEBUG [RS:0;dfce6ce5b10c:38861 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T16:30:42,418 DEBUG [RS:2;dfce6ce5b10c:34709 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T16:30:42,418 DEBUG [RS:1;dfce6ce5b10c:32855 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T16:30:42,418 DEBUG [RS:1;dfce6ce5b10c:32855 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T16:30:42,418 DEBUG [RS:0;dfce6ce5b10c:38861 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T16:30:42,419 INFO [RS:1;dfce6ce5b10c:32855 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T16:30:42,419 INFO [RS:0;dfce6ce5b10c:38861 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T16:30:42,419 INFO [RS:1;dfce6ce5b10c:32855 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T16:30:42,419 INFO [RS:0;dfce6ce5b10c:38861 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T16:30:42,419 DEBUG [RS:2;dfce6ce5b10c:34709 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T16:30:42,419 INFO [RS:2;dfce6ce5b10c:34709 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T16:30:42,419 INFO [RS:2;dfce6ce5b10c:34709 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T16:30:42,524 INFO [RS:0;dfce6ce5b10c:38861 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T16:30:42,524 INFO [RS:1;dfce6ce5b10c:32855 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T16:30:42,524 INFO [RS:2;dfce6ce5b10c:34709 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T16:30:42,527 INFO [RS:1;dfce6ce5b10c:32855 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dfce6ce5b10c%2C32855%2C1733761840798, suffix=, logDir=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,32855,1733761840798, archiveDir=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/oldWALs, maxLogs=32 2024-12-09T16:30:42,527 INFO [RS:2;dfce6ce5b10c:34709 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dfce6ce5b10c%2C34709%2C1733761840861, suffix=, logDir=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,34709,1733761840861, archiveDir=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/oldWALs, maxLogs=32 2024-12-09T16:30:42,527 INFO [RS:0;dfce6ce5b10c:38861 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dfce6ce5b10c%2C38861%2C1733761840646, suffix=, logDir=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,38861,1733761840646, archiveDir=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/oldWALs, maxLogs=32 2024-12-09T16:30:42,544 DEBUG [RS:2;dfce6ce5b10c:34709 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,34709,1733761840861/dfce6ce5b10c%2C34709%2C1733761840861.1733761842531, exclude list is [], retry=0 2024-12-09T16:30:42,544 DEBUG [RS:1;dfce6ce5b10c:32855 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,32855,1733761840798/dfce6ce5b10c%2C32855%2C1733761840798.1733761842531, exclude list is [], retry=0 2024-12-09T16:30:42,545 DEBUG [RS:0;dfce6ce5b10c:38861 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,38861,1733761840646/dfce6ce5b10c%2C38861%2C1733761840646.1733761842531, exclude list is [], retry=0 2024-12-09T16:30:42,549 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:30:42,549 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:30:42,550 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:30:42,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:30:42,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:30:42,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:30:42,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:30:42,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:30:42,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:30:42,562 INFO [RS:2;dfce6ce5b10c:34709 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,34709,1733761840861/dfce6ce5b10c%2C34709%2C1733761840861.1733761842531 2024-12-09T16:30:42,563 INFO [RS:1;dfce6ce5b10c:32855 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,32855,1733761840798/dfce6ce5b10c%2C32855%2C1733761840798.1733761842531 2024-12-09T16:30:42,564 DEBUG [RS:2;dfce6ce5b10c:34709 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:39661:39661),(127.0.0.1/127.0.0.1:35519:35519)] 2024-12-09T16:30:42,564 INFO [RS:0;dfce6ce5b10c:38861 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,38861,1733761840646/dfce6ce5b10c%2C38861%2C1733761840646.1733761842531 2024-12-09T16:30:42,565 DEBUG [RS:1;dfce6ce5b10c:32855 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:39661:39661)] 2024-12-09T16:30:42,565 DEBUG [RS:0;dfce6ce5b10c:38861 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:39661:39661)] 2024-12-09T16:30:42,641 DEBUG [dfce6ce5b10c:35827 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-09T16:30:42,652 DEBUG [dfce6ce5b10c:35827 {}] balancer.BalancerClusterState(204): Hosts are {dfce6ce5b10c=0} racks are {/default-rack=0} 2024-12-09T16:30:42,661 DEBUG [dfce6ce5b10c:35827 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T16:30:42,661 DEBUG [dfce6ce5b10c:35827 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T16:30:42,661 DEBUG [dfce6ce5b10c:35827 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-09T16:30:42,661 DEBUG [dfce6ce5b10c:35827 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T16:30:42,663 DEBUG [dfce6ce5b10c:35827 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T16:30:42,663 DEBUG [dfce6ce5b10c:35827 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-09T16:30:42,663 INFO [dfce6ce5b10c:35827 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T16:30:42,663 INFO [dfce6ce5b10c:35827 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T16:30:42,663 INFO [dfce6ce5b10c:35827 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-09T16:30:42,663 DEBUG [dfce6ce5b10c:35827 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T16:30:42,673 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=dfce6ce5b10c,38861,1733761840646 2024-12-09T16:30:42,678 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dfce6ce5b10c,38861,1733761840646, state=OPENING 2024-12-09T16:30:42,845 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T16:30:42,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38861-0x100bfc42e970001, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:30:42,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:30:42,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32855-0x100bfc42e970002, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:30:42,987 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34709-0x100bfc42e970003, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:30:42,989 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T16:30:42,989 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T16:30:42,989 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T16:30:42,989 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T16:30:42,992 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T16:30:42,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=dfce6ce5b10c,38861,1733761840646}] 2024-12-09T16:30:43,176 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T16:30:43,179 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41173, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T16:30:43,194 INFO [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T16:30:43,195 INFO [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:30:43,196 INFO [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T16:30:43,201 INFO [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dfce6ce5b10c%2C38861%2C1733761840646.meta, suffix=.meta, logDir=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,38861,1733761840646, archiveDir=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/oldWALs, maxLogs=32 2024-12-09T16:30:43,225 DEBUG [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,38861,1733761840646/dfce6ce5b10c%2C38861%2C1733761840646.meta.1733761843203.meta, exclude list is [], retry=0 2024-12-09T16:30:43,229 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:30:43,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:30:43,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:30:43,233 INFO [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,38861,1733761840646/dfce6ce5b10c%2C38861%2C1733761840646.meta.1733761843203.meta 2024-12-09T16:30:43,233 DEBUG [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:39661:39661)] 2024-12-09T16:30:43,234 DEBUG [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T16:30:43,236 DEBUG [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T16:30:43,239 DEBUG [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T16:30:43,245 INFO [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T16:30:43,249 DEBUG [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T16:30:43,250 DEBUG [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:30:43,250 DEBUG [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T16:30:43,250 DEBUG [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T16:30:43,254 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T16:30:43,256 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T16:30:43,256 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:43,257 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T16:30:43,258 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T16:30:43,260 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T16:30:43,260 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:43,261 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T16:30:43,262 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T16:30:43,264 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T16:30:43,264 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:43,265 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T16:30:43,265 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T16:30:43,267 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T16:30:43,267 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:43,268 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T16:30:43,269 DEBUG [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T16:30:43,270 DEBUG [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/hbase/meta/1588230740 2024-12-09T16:30:43,274 DEBUG [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/hbase/meta/1588230740 2024-12-09T16:30:43,278 DEBUG [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T16:30:43,278 DEBUG [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T16:30:43,279 DEBUG [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T16:30:43,282 DEBUG [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T16:30:43,284 INFO [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74413456, jitterRate=0.10884690284729004}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T16:30:43,284 DEBUG [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T16:30:43,286 DEBUG [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733761843251Writing region info on filesystem at 1733761843251Initializing all the Stores at 1733761843253 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733761843253Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733761843253Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761843253Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733761843253Cleaning up temporary data from old regions at 1733761843278 (+25 ms)Running coprocessor post-open hooks at 1733761843285 (+7 ms)Region opened successfully at 1733761843286 (+1 ms) 2024-12-09T16:30:43,293 INFO [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733761843163 2024-12-09T16:30:43,305 DEBUG [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T16:30:43,305 INFO [RS_OPEN_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T16:30:43,307 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=dfce6ce5b10c,38861,1733761840646 2024-12-09T16:30:43,309 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dfce6ce5b10c,38861,1733761840646, state=OPEN 2024-12-09T16:30:43,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T16:30:43,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38861-0x100bfc42e970001, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T16:30:43,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32855-0x100bfc42e970002, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T16:30:43,596 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34709-0x100bfc42e970003, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T16:30:43,596 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T16:30:43,596 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T16:30:43,596 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T16:30:43,596 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T16:30:43,597 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=dfce6ce5b10c,38861,1733761840646 2024-12-09T16:30:43,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T16:30:43,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=dfce6ce5b10c,38861,1733761840646 in 603 msec 2024-12-09T16:30:43,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T16:30:43,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.3820 sec 2024-12-09T16:30:43,611 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T16:30:43,611 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-09T16:30:43,631 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T16:30:43,633 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dfce6ce5b10c,38861,1733761840646, seqNum=-1] 2024-12-09T16:30:43,659 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T16:30:43,661 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55893, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T16:30:43,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7230 sec 2024-12-09T16:30:43,685 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733761843685, completionTime=-1 2024-12-09T16:30:43,688 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-09T16:30:43,688 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T16:30:43,735 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-09T16:30:43,735 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733761903735 2024-12-09T16:30:43,735 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733761963735 2024-12-09T16:30:43,735 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 47 msec 2024-12-09T16:30:43,741 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-09T16:30:43,753 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dfce6ce5b10c,35827,1733761839880-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:43,753 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dfce6ce5b10c,35827,1733761839880-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:43,753 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dfce6ce5b10c,35827,1733761839880-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:43,754 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-dfce6ce5b10c:35827, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:43,755 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:43,755 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:43,760 DEBUG [master/dfce6ce5b10c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-09T16:30:43,781 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.810sec 2024-12-09T16:30:43,783 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T16:30:43,785 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T16:30:43,786 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T16:30:43,786 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T16:30:43,786 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T16:30:43,787 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dfce6ce5b10c,35827,1733761839880-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T16:30:43,788 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dfce6ce5b10c,35827,1733761839880-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T16:30:43,792 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-09T16:30:43,793 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T16:30:43,794 INFO [master/dfce6ce5b10c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dfce6ce5b10c,35827,1733761839880-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:43,821 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ef44f44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T16:30:43,822 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request dfce6ce5b10c,35827,-1 for getting cluster id 2024-12-09T16:30:43,824 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T16:30:43,833 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80e51bf9-2430-42cd-8179-eacd4b0999ae' 2024-12-09T16:30:43,836 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T16:30:43,836 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80e51bf9-2430-42cd-8179-eacd4b0999ae" 2024-12-09T16:30:43,836 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f89c390, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T16:30:43,836 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [dfce6ce5b10c,35827,-1] 2024-12-09T16:30:43,839 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T16:30:43,841 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T16:30:43,842 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T16:30:43,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d4a9819, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T16:30:43,846 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T16:30:43,853 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dfce6ce5b10c,38861,1733761840646, seqNum=-1] 2024-12-09T16:30:43,853 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T16:30:43,855 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T16:30:43,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=dfce6ce5b10c,35827,1733761839880 2024-12-09T16:30:43,875 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:39607/hbase 2024-12-09T16:30:43,894 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=367, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=703, ProcessCount=11, AvailableMemoryMB=6718 2024-12-09T16:30:43,919 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:30:43,923 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:30:43,924 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T16:30:43,929 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-54939348, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/hregion-54939348, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:30:43,944 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-54939348/hregion-54939348.1733761843931, exclude list is [], retry=0 2024-12-09T16:30:43,949 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:30:43,949 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:30:43,950 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:30:43,953 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-54939348/hregion-54939348.1733761843931 2024-12-09T16:30:43,954 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39661:39661),(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:37099:37099)] 2024-12-09T16:30:43,954 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1b6a6684ce15034fd73d9966f03b6fcc, NAME => 'testReplayEditsWrittenIntoWAL,,1733761843920.1b6a6684ce15034fd73d9966f03b6fcc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39607/hbase 2024-12-09T16:30:43,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741838_1014 (size=64) 2024-12-09T16:30:43,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741838_1014 (size=64) 2024-12-09T16:30:43,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741838_1014 (size=64) 2024-12-09T16:30:43,975 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733761843920.1b6a6684ce15034fd73d9966f03b6fcc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:30:43,977 INFO [StoreOpener-1b6a6684ce15034fd73d9966f03b6fcc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1b6a6684ce15034fd73d9966f03b6fcc 2024-12-09T16:30:43,980 INFO [StoreOpener-1b6a6684ce15034fd73d9966f03b6fcc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b6a6684ce15034fd73d9966f03b6fcc columnFamilyName a 2024-12-09T16:30:43,980 DEBUG [StoreOpener-1b6a6684ce15034fd73d9966f03b6fcc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:43,981 INFO [StoreOpener-1b6a6684ce15034fd73d9966f03b6fcc-1 {}] regionserver.HStore(327): Store=1b6a6684ce15034fd73d9966f03b6fcc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:43,981 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1b6a6684ce15034fd73d9966f03b6fcc 2024-12-09T16:30:43,982 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc 2024-12-09T16:30:43,983 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc 2024-12-09T16:30:43,983 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1b6a6684ce15034fd73d9966f03b6fcc 2024-12-09T16:30:43,983 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1b6a6684ce15034fd73d9966f03b6fcc 2024-12-09T16:30:43,986 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1b6a6684ce15034fd73d9966f03b6fcc 2024-12-09T16:30:43,990 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T16:30:43,991 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1b6a6684ce15034fd73d9966f03b6fcc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59611878, jitterRate=-0.11171379685401917}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T16:30:43,992 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1b6a6684ce15034fd73d9966f03b6fcc: Writing region info on filesystem at 1733761843975Initializing all the Stores at 1733761843977 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761843977Cleaning up temporary data from old regions at 1733761843983 (+6 ms)Region opened successfully at 1733761843992 (+9 ms) 2024-12-09T16:30:43,993 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1b6a6684ce15034fd73d9966f03b6fcc, disabling compactions & flushes 2024-12-09T16:30:43,993 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733761843920.1b6a6684ce15034fd73d9966f03b6fcc. 2024-12-09T16:30:43,993 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733761843920.1b6a6684ce15034fd73d9966f03b6fcc. 2024-12-09T16:30:43,993 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733761843920.1b6a6684ce15034fd73d9966f03b6fcc. after waiting 0 ms 2024-12-09T16:30:43,993 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733761843920.1b6a6684ce15034fd73d9966f03b6fcc. 2024-12-09T16:30:43,994 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733761843920.1b6a6684ce15034fd73d9966f03b6fcc. 2024-12-09T16:30:43,994 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1b6a6684ce15034fd73d9966f03b6fcc: Waiting for close lock at 1733761843993Disabling compacts and flushes for region at 1733761843993Disabling writes for close at 1733761843993Writing region close event to WAL at 1733761843994 (+1 ms)Closed at 1733761843994 2024-12-09T16:30:44,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741837_1013 (size=93) 2024-12-09T16:30:44,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741837_1013 (size=93) 2024-12-09T16:30:44,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741837_1013 (size=93) 2024-12-09T16:30:44,012 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T16:30:44,012 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-54939348:(num 1733761843931) 2024-12-09T16:30:44,015 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T16:30:44,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741839_1015 (size=276) 2024-12-09T16:30:44,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741839_1015 (size=276) 2024-12-09T16:30:44,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741839_1015 (size=276) 2024-12-09T16:30:44,038 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T16:30:44,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741840_1016 (size=230) 2024-12-09T16:30:44,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741840_1016 (size=230) 2024-12-09T16:30:44,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741840_1016 (size=230) 2024-12-09T16:30:44,072 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal-1, size=276 (276bytes) 2024-12-09T16:30:44,073 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-09T16:30:44,073 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-09T16:30:44,073 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal-1 2024-12-09T16:30:44,079 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal-1 after 3ms 2024-12-09T16:30:44,085 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:30:44,085 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal-1 took 14ms 2024-12-09T16:30:44,090 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal-1 so closing down 2024-12-09T16:30:44,093 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-09T16:30:44,096 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T16:30:44,096 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc/recovered.edits/0000000000000000001-wal-1.temp 2024-12-09T16:30:44,097 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T16:30:44,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741841_1017 (size=276) 2024-12-09T16:30:44,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741841_1017 (size=276) 2024-12-09T16:30:44,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741841_1017 (size=276) 2024-12-09T16:30:44,111 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-09T16:30:44,115 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc/recovered.edits/0000000000000000002 2024-12-09T16:30:44,118 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 28 ms; skipped=0; WAL=hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-12-09T16:30:44,118 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal-1, journal: Splitting hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal-1, size=276 (276bytes) at 1733761844073Finishing writing output for hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal-1 so closing down at 1733761844090 (+17 ms)Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc/recovered.edits/0000000000000000001-wal-1.temp at 1733761844096 (+6 ms)3 split writer threads finished at 1733761844097 (+1 ms)Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733761844111 (+14 ms)Rename recovered edits hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc/recovered.edits/0000000000000000002 at 1733761844115 (+4 ms)Processed 2 edits across 1 Regions in 28 ms; skipped=0; WAL=hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1733761844118 (+3 ms) 2024-12-09T16:30:44,133 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal-2, size=230 (230bytes) 2024-12-09T16:30:44,133 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal-2 2024-12-09T16:30:44,135 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal-2 after 1ms 2024-12-09T16:30:44,140 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:30:44,140 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal-2 took 7ms 2024-12-09T16:30:44,143 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal-2 so closing down 2024-12-09T16:30:44,143 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T16:30:44,147 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-09T16:30:44,148 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc/recovered.edits/0000000000000000002-wal-2.temp 2024-12-09T16:30:44,149 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T16:30:44,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741842_1018 (size=230) 2024-12-09T16:30:44,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741842_1018 (size=230) 2024-12-09T16:30:44,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741842_1018 (size=230) 2024-12-09T16:30:44,161 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-09T16:30:44,167 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:30:44,170 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc/recovered.edits/0000000000000000002-wal-2.temp, length=230 2024-12-09T16:30:44,171 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 30 ms; skipped=0; WAL=hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-12-09T16:30:44,171 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal-2, journal: Splitting hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal-2, size=230 (230bytes) at 1733761844133Finishing writing output for hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal-2 so closing down at 1733761844143 (+10 ms)Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc/recovered.edits/0000000000000000002-wal-2.temp at 1733761844149 (+6 ms)3 split writer threads finished at 1733761844149Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733761844161 (+12 ms)Processed 1 edits across 1 Regions in 30 ms; skipped=0; WAL=hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1733761844171 (+10 ms) 2024-12-09T16:30:44,172 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T16:30:44,174 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:30:44,188 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal.1733761844175, exclude list is [], retry=0 2024-12-09T16:30:44,193 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:30:44,194 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:30:44,194 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:30:44,197 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal.1733761844175 2024-12-09T16:30:44,198 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39661:39661),(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:35519:35519)] 2024-12-09T16:30:44,199 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1b6a6684ce15034fd73d9966f03b6fcc, NAME => 'testReplayEditsWrittenIntoWAL,,1733761843920.1b6a6684ce15034fd73d9966f03b6fcc.', STARTKEY => '', ENDKEY => ''} 2024-12-09T16:30:44,199 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733761843920.1b6a6684ce15034fd73d9966f03b6fcc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:30:44,199 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1b6a6684ce15034fd73d9966f03b6fcc 2024-12-09T16:30:44,199 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1b6a6684ce15034fd73d9966f03b6fcc 2024-12-09T16:30:44,202 INFO [StoreOpener-1b6a6684ce15034fd73d9966f03b6fcc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1b6a6684ce15034fd73d9966f03b6fcc 2024-12-09T16:30:44,203 INFO [StoreOpener-1b6a6684ce15034fd73d9966f03b6fcc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b6a6684ce15034fd73d9966f03b6fcc columnFamilyName a 2024-12-09T16:30:44,204 DEBUG [StoreOpener-1b6a6684ce15034fd73d9966f03b6fcc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:44,205 INFO [StoreOpener-1b6a6684ce15034fd73d9966f03b6fcc-1 {}] regionserver.HStore(327): Store=1b6a6684ce15034fd73d9966f03b6fcc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:44,205 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1b6a6684ce15034fd73d9966f03b6fcc 2024-12-09T16:30:44,207 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc 2024-12-09T16:30:44,210 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc 2024-12-09T16:30:44,212 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc/recovered.edits/0000000000000000002 2024-12-09T16:30:44,216 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:30:44,222 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc/recovered.edits/0000000000000000002 2024-12-09T16:30:44,226 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1b6a6684ce15034fd73d9966f03b6fcc 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-09T16:30:44,290 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc/.tmp/a/3d56b612acaa4eff969290b351fb4e56 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733761844012/Put/seqid=0 2024-12-09T16:30:44,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741844_1020 (size=5170) 2024-12-09T16:30:44,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741844_1020 (size=5170) 2024-12-09T16:30:44,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741844_1020 (size=5170) 2024-12-09T16:30:44,315 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc/.tmp/a/3d56b612acaa4eff969290b351fb4e56 2024-12-09T16:30:44,369 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc/.tmp/a/3d56b612acaa4eff969290b351fb4e56 as hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc/a/3d56b612acaa4eff969290b351fb4e56 2024-12-09T16:30:44,380 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc/a/3d56b612acaa4eff969290b351fb4e56, entries=2, sequenceid=2, filesize=5.0 K 2024-12-09T16:30:44,386 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 1b6a6684ce15034fd73d9966f03b6fcc in 159ms, sequenceid=2, compaction requested=false; wal=null 2024-12-09T16:30:44,387 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc/recovered.edits/0000000000000000002 2024-12-09T16:30:44,388 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1b6a6684ce15034fd73d9966f03b6fcc 2024-12-09T16:30:44,388 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1b6a6684ce15034fd73d9966f03b6fcc 2024-12-09T16:30:44,391 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1b6a6684ce15034fd73d9966f03b6fcc 2024-12-09T16:30:44,395 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/1b6a6684ce15034fd73d9966f03b6fcc/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-09T16:30:44,397 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1b6a6684ce15034fd73d9966f03b6fcc; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71077771, jitterRate=0.05914132297039032}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T16:30:44,398 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1b6a6684ce15034fd73d9966f03b6fcc: Writing region info on filesystem at 1733761844199Initializing all the Stores at 1733761844201 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761844201Obtaining lock to block concurrent updates at 1733761844226 (+25 ms)Preparing flush snapshotting stores in 1b6a6684ce15034fd73d9966f03b6fcc at 1733761844226Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733761843920.1b6a6684ce15034fd73d9966f03b6fcc., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733761844229 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733761843920.1b6a6684ce15034fd73d9966f03b6fcc. at 1733761844229Flushing 1b6a6684ce15034fd73d9966f03b6fcc/a: creating writer at 1733761844231 (+2 ms)Flushing 1b6a6684ce15034fd73d9966f03b6fcc/a: appending metadata at 1733761844278 (+47 ms)Flushing 1b6a6684ce15034fd73d9966f03b6fcc/a: closing flushed file at 1733761844281 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30b35044: reopening flushed file at 1733761844366 (+85 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 1b6a6684ce15034fd73d9966f03b6fcc in 159ms, sequenceid=2, compaction requested=false; wal=null at 1733761844386 (+20 ms)Cleaning up temporary data from old regions at 1733761844388 (+2 ms)Region opened successfully at 1733761844398 (+10 ms) 2024-12-09T16:30:44,427 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=378 (was 367) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:55462 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-427817209-172.17.0.2-1733761835089:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:39607/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:55318 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:41956 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-427817209-172.17.0.2-1733761835089:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-427817209-172.17.0.2-1733761835089:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:50482 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:50630 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=699 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=703 (was 703), ProcessCount=11 (was 11), AvailableMemoryMB=6851 (was 6718) - AvailableMemoryMB LEAK? - 2024-12-09T16:30:44,440 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=378, OpenFileDescriptor=699, MaxFileDescriptor=1048576, SystemLoadAverage=703, ProcessCount=11, AvailableMemoryMB=6850 2024-12-09T16:30:44,456 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:30:44,459 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:30:44,460 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T16:30:44,465 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-41784444, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/hregion-41784444, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:30:44,481 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-41784444/hregion-41784444.1733761844466, exclude list is [], retry=0 2024-12-09T16:30:44,485 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:30:44,485 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:30:44,488 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:30:44,491 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-41784444/hregion-41784444.1733761844466 2024-12-09T16:30:44,492 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39661:39661),(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:37099:37099)] 2024-12-09T16:30:44,492 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 7527f607577e0238edbe41f457bfcb2d, NAME => 'testReplayEditsWrittenIntoWAL,,1733761844457.7527f607577e0238edbe41f457bfcb2d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39607/hbase 2024-12-09T16:30:44,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741846_1022 (size=64) 2024-12-09T16:30:44,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741846_1022 (size=64) 2024-12-09T16:30:44,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741846_1022 (size=64) 2024-12-09T16:30:44,511 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733761844457.7527f607577e0238edbe41f457bfcb2d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:30:44,514 INFO [StoreOpener-7527f607577e0238edbe41f457bfcb2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7527f607577e0238edbe41f457bfcb2d 2024-12-09T16:30:44,516 INFO [StoreOpener-7527f607577e0238edbe41f457bfcb2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7527f607577e0238edbe41f457bfcb2d columnFamilyName a 2024-12-09T16:30:44,517 DEBUG [StoreOpener-7527f607577e0238edbe41f457bfcb2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:44,517 INFO [StoreOpener-7527f607577e0238edbe41f457bfcb2d-1 {}] regionserver.HStore(327): Store=7527f607577e0238edbe41f457bfcb2d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:44,518 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7527f607577e0238edbe41f457bfcb2d 2024-12-09T16:30:44,519 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d 2024-12-09T16:30:44,520 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d 2024-12-09T16:30:44,520 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7527f607577e0238edbe41f457bfcb2d 2024-12-09T16:30:44,520 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7527f607577e0238edbe41f457bfcb2d 2024-12-09T16:30:44,523 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7527f607577e0238edbe41f457bfcb2d 2024-12-09T16:30:44,527 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T16:30:44,528 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7527f607577e0238edbe41f457bfcb2d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71225273, jitterRate=0.06133927404880524}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T16:30:44,528 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7527f607577e0238edbe41f457bfcb2d: Writing region info on filesystem at 1733761844511Initializing all the Stores at 1733761844513 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761844513Cleaning up temporary data from old regions at 1733761844521 (+8 ms)Region opened successfully at 1733761844528 (+7 ms) 2024-12-09T16:30:44,528 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7527f607577e0238edbe41f457bfcb2d, disabling compactions & flushes 2024-12-09T16:30:44,528 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733761844457.7527f607577e0238edbe41f457bfcb2d. 2024-12-09T16:30:44,528 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733761844457.7527f607577e0238edbe41f457bfcb2d. 2024-12-09T16:30:44,528 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733761844457.7527f607577e0238edbe41f457bfcb2d. after waiting 0 ms 2024-12-09T16:30:44,528 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733761844457.7527f607577e0238edbe41f457bfcb2d. 2024-12-09T16:30:44,529 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733761844457.7527f607577e0238edbe41f457bfcb2d. 2024-12-09T16:30:44,529 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7527f607577e0238edbe41f457bfcb2d: Waiting for close lock at 1733761844528Disabling compacts and flushes for region at 1733761844528Disabling writes for close at 1733761844528Writing region close event to WAL at 1733761844529 (+1 ms)Closed at 1733761844529 2024-12-09T16:30:44,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741845_1021 (size=93) 2024-12-09T16:30:44,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741845_1021 (size=93) 2024-12-09T16:30:44,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741845_1021 (size=93) 2024-12-09T16:30:44,539 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T16:30:44,539 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-41784444:(num 1733761844466) 2024-12-09T16:30:44,540 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T16:30:44,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741847_1023 (size=276) 2024-12-09T16:30:44,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741847_1023 (size=276) 2024-12-09T16:30:44,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741847_1023 (size=276) 2024-12-09T16:30:44,558 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T16:30:44,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741848_1024 (size=230) 2024-12-09T16:30:44,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741848_1024 (size=230) 2024-12-09T16:30:44,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741848_1024 (size=230) 2024-12-09T16:30:44,592 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal-2, size=230 (230bytes) 2024-12-09T16:30:44,592 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal-2 2024-12-09T16:30:44,593 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal-2 after 1ms 2024-12-09T16:30:44,597 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:30:44,597 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal-2 took 5ms 2024-12-09T16:30:44,600 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal-2 so closing down 2024-12-09T16:30:44,600 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T16:30:44,602 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-09T16:30:44,604 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/recovered.edits/0000000000000000002-wal-2.temp 2024-12-09T16:30:44,604 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T16:30:44,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741849_1025 (size=230) 2024-12-09T16:30:44,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741849_1025 (size=230) 2024-12-09T16:30:44,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741849_1025 (size=230) 2024-12-09T16:30:44,615 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-09T16:30:44,617 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/recovered.edits/0000000000000000002 2024-12-09T16:30:44,617 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-12-09T16:30:44,617 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal-2, journal: Splitting hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal-2, size=230 (230bytes) at 1733761844592Finishing writing output for hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal-2 so closing down at 1733761844600 (+8 ms)Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/recovered.edits/0000000000000000002-wal-2.temp at 1733761844604 (+4 ms)3 split writer threads finished at 1733761844604Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733761844615 (+11 ms)Rename recovered edits hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/recovered.edits/0000000000000000002 at 1733761844617 (+2 ms)Processed 1 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1733761844617 2024-12-09T16:30:44,639 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal-1, size=276 (276bytes) 2024-12-09T16:30:44,639 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal-1 2024-12-09T16:30:44,640 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal-1 after 1ms 2024-12-09T16:30:44,645 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:30:44,645 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal-1 took 6ms 2024-12-09T16:30:44,649 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal-1 so closing down 2024-12-09T16:30:44,649 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T16:30:44,652 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-09T16:30:44,654 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/recovered.edits/0000000000000000001-wal-1.temp 2024-12-09T16:30:44,655 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T16:30:44,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741850_1026 (size=276) 2024-12-09T16:30:44,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741850_1026 (size=276) 2024-12-09T16:30:44,670 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-09T16:30:44,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741850_1026 (size=276) 2024-12-09T16:30:44,676 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:30:44,680 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/recovered.edits/0000000000000000002, length=230 2024-12-09T16:30:44,683 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/recovered.edits/0000000000000000002 2024-12-09T16:30:44,683 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 37 ms; skipped=0; WAL=hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-12-09T16:30:44,684 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal-1, journal: Splitting hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal-1, size=276 (276bytes) at 1733761844639Finishing writing output for hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal-1 so closing down at 1733761844649 (+10 ms)Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/recovered.edits/0000000000000000001-wal-1.temp at 1733761844654 (+5 ms)3 split writer threads finished at 1733761844655 (+1 ms)Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733761844670 (+15 ms)Rename recovered edits hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/recovered.edits/0000000000000000002 at 1733761844683 (+13 ms)Processed 2 edits across 1 Regions in 37 ms; skipped=0; WAL=hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1733761844683 2024-12-09T16:30:44,684 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T16:30:44,687 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:30:44,707 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal.1733761844688, exclude list is [], retry=0 2024-12-09T16:30:44,712 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:30:44,713 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:30:44,713 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:30:44,718 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal.1733761844688 2024-12-09T16:30:44,722 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:39661:39661)] 2024-12-09T16:30:44,722 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7527f607577e0238edbe41f457bfcb2d, NAME => 'testReplayEditsWrittenIntoWAL,,1733761844457.7527f607577e0238edbe41f457bfcb2d.', STARTKEY => '', ENDKEY => ''} 2024-12-09T16:30:44,722 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733761844457.7527f607577e0238edbe41f457bfcb2d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:30:44,722 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7527f607577e0238edbe41f457bfcb2d 2024-12-09T16:30:44,722 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7527f607577e0238edbe41f457bfcb2d 2024-12-09T16:30:44,725 INFO [StoreOpener-7527f607577e0238edbe41f457bfcb2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7527f607577e0238edbe41f457bfcb2d 2024-12-09T16:30:44,728 INFO [StoreOpener-7527f607577e0238edbe41f457bfcb2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7527f607577e0238edbe41f457bfcb2d columnFamilyName a 2024-12-09T16:30:44,728 DEBUG [StoreOpener-7527f607577e0238edbe41f457bfcb2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:44,729 INFO [StoreOpener-7527f607577e0238edbe41f457bfcb2d-1 {}] regionserver.HStore(327): Store=7527f607577e0238edbe41f457bfcb2d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:44,729 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7527f607577e0238edbe41f457bfcb2d 2024-12-09T16:30:44,730 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d 2024-12-09T16:30:44,733 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d 2024-12-09T16:30:44,734 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/recovered.edits/0000000000000000002 2024-12-09T16:30:44,738 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:30:44,740 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/recovered.edits/0000000000000000002 2024-12-09T16:30:44,741 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7527f607577e0238edbe41f457bfcb2d 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-09T16:30:44,759 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/.tmp/a/9dd0270b1cb24f7997cc9e6e9a24f69c is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733761844539/Put/seqid=0 2024-12-09T16:30:44,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741852_1028 (size=5170) 2024-12-09T16:30:44,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741852_1028 (size=5170) 2024-12-09T16:30:44,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741852_1028 (size=5170) 2024-12-09T16:30:44,770 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/.tmp/a/9dd0270b1cb24f7997cc9e6e9a24f69c 2024-12-09T16:30:44,782 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/.tmp/a/9dd0270b1cb24f7997cc9e6e9a24f69c as hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/a/9dd0270b1cb24f7997cc9e6e9a24f69c 2024-12-09T16:30:44,792 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/a/9dd0270b1cb24f7997cc9e6e9a24f69c, entries=2, sequenceid=2, filesize=5.0 K 2024-12-09T16:30:44,792 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 7527f607577e0238edbe41f457bfcb2d in 51ms, sequenceid=2, compaction requested=false; wal=null 2024-12-09T16:30:44,793 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/recovered.edits/0000000000000000002 2024-12-09T16:30:44,794 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7527f607577e0238edbe41f457bfcb2d 2024-12-09T16:30:44,794 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7527f607577e0238edbe41f457bfcb2d 2024-12-09T16:30:44,797 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7527f607577e0238edbe41f457bfcb2d 2024-12-09T16:30:44,800 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/7527f607577e0238edbe41f457bfcb2d/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-09T16:30:44,801 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7527f607577e0238edbe41f457bfcb2d; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73398048, jitterRate=0.09371614456176758}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T16:30:44,802 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7527f607577e0238edbe41f457bfcb2d: Writing region info on filesystem at 1733761844723Initializing all the Stores at 1733761844724 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761844725 (+1 ms)Obtaining lock to block concurrent updates at 1733761844741 (+16 ms)Preparing flush snapshotting stores in 7527f607577e0238edbe41f457bfcb2d at 1733761844741Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733761844457.7527f607577e0238edbe41f457bfcb2d., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733761844741Flushing stores of testReplayEditsWrittenIntoWAL,,1733761844457.7527f607577e0238edbe41f457bfcb2d. at 1733761844741Flushing 7527f607577e0238edbe41f457bfcb2d/a: creating writer at 1733761844741Flushing 7527f607577e0238edbe41f457bfcb2d/a: appending metadata at 1733761844758 (+17 ms)Flushing 7527f607577e0238edbe41f457bfcb2d/a: closing flushed file at 1733761844758Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3499622a: reopening flushed file at 1733761844780 (+22 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 7527f607577e0238edbe41f457bfcb2d in 51ms, sequenceid=2, compaction requested=false; wal=null at 1733761844792 (+12 ms)Cleaning up temporary data from old regions at 1733761844794 (+2 ms)Region opened successfully at 1733761844802 (+8 ms) 2024-12-09T16:30:44,821 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=389 (was 378) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:42054 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-427817209-172.17.0.2-1733761835089:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:50690 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:50482 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:55506 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:42014 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-427817209-172.17.0.2-1733761835089:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-427817209-172.17.0.2-1733761835089:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:55318 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 699) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=703 (was 703), ProcessCount=11 (was 11), AvailableMemoryMB=6842 (was 6850) 2024-12-09T16:30:44,835 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=389, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=703, ProcessCount=11, AvailableMemoryMB=6841 2024-12-09T16:30:44,858 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:30:44,861 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:30:44,863 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T16:30:44,867 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-81573533, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/hregion-81573533, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:30:44,887 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-81573533/hregion-81573533.1733761844868, exclude list is [], retry=0 2024-12-09T16:30:44,892 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:30:44,892 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:30:44,893 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:30:44,896 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-81573533/hregion-81573533.1733761844868 2024-12-09T16:30:44,897 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:39661:39661)] 2024-12-09T16:30:44,897 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => bb9e94e1a90f12ab562e74f9c6ecaf2d, NAME => 'testReplayEditsWrittenIntoWAL,,1733761844859.bb9e94e1a90f12ab562e74f9c6ecaf2d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39607/hbase 2024-12-09T16:30:44,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741854_1030 (size=64) 2024-12-09T16:30:44,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741854_1030 (size=64) 2024-12-09T16:30:44,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741854_1030 (size=64) 2024-12-09T16:30:44,911 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733761844859.bb9e94e1a90f12ab562e74f9c6ecaf2d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:30:44,913 INFO [StoreOpener-bb9e94e1a90f12ab562e74f9c6ecaf2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bb9e94e1a90f12ab562e74f9c6ecaf2d 2024-12-09T16:30:44,915 INFO [StoreOpener-bb9e94e1a90f12ab562e74f9c6ecaf2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb9e94e1a90f12ab562e74f9c6ecaf2d columnFamilyName a 2024-12-09T16:30:44,915 DEBUG [StoreOpener-bb9e94e1a90f12ab562e74f9c6ecaf2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:44,916 INFO [StoreOpener-bb9e94e1a90f12ab562e74f9c6ecaf2d-1 {}] regionserver.HStore(327): Store=bb9e94e1a90f12ab562e74f9c6ecaf2d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:44,916 INFO [StoreOpener-bb9e94e1a90f12ab562e74f9c6ecaf2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bb9e94e1a90f12ab562e74f9c6ecaf2d 2024-12-09T16:30:44,919 INFO [StoreOpener-bb9e94e1a90f12ab562e74f9c6ecaf2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb9e94e1a90f12ab562e74f9c6ecaf2d columnFamilyName b 2024-12-09T16:30:44,919 DEBUG [StoreOpener-bb9e94e1a90f12ab562e74f9c6ecaf2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:44,920 INFO [StoreOpener-bb9e94e1a90f12ab562e74f9c6ecaf2d-1 {}] regionserver.HStore(327): Store=bb9e94e1a90f12ab562e74f9c6ecaf2d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:44,920 INFO [StoreOpener-bb9e94e1a90f12ab562e74f9c6ecaf2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bb9e94e1a90f12ab562e74f9c6ecaf2d 2024-12-09T16:30:44,923 INFO [StoreOpener-bb9e94e1a90f12ab562e74f9c6ecaf2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb9e94e1a90f12ab562e74f9c6ecaf2d columnFamilyName c 2024-12-09T16:30:44,923 DEBUG [StoreOpener-bb9e94e1a90f12ab562e74f9c6ecaf2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:44,924 INFO [StoreOpener-bb9e94e1a90f12ab562e74f9c6ecaf2d-1 {}] regionserver.HStore(327): Store=bb9e94e1a90f12ab562e74f9c6ecaf2d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:44,924 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bb9e94e1a90f12ab562e74f9c6ecaf2d 2024-12-09T16:30:44,926 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d 2024-12-09T16:30:44,926 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d 2024-12-09T16:30:44,928 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bb9e94e1a90f12ab562e74f9c6ecaf2d 2024-12-09T16:30:44,928 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bb9e94e1a90f12ab562e74f9c6ecaf2d 2024-12-09T16:30:44,929 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T16:30:44,930 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bb9e94e1a90f12ab562e74f9c6ecaf2d 2024-12-09T16:30:44,933 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T16:30:44,934 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bb9e94e1a90f12ab562e74f9c6ecaf2d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65110506, jitterRate=-0.029777854681015015}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T16:30:44,935 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bb9e94e1a90f12ab562e74f9c6ecaf2d: Writing region info on filesystem at 1733761844911Initializing all the Stores at 1733761844912 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761844912Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761844913 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761844913Cleaning up temporary data from old regions at 1733761844928 (+15 ms)Region opened successfully at 1733761844934 (+6 ms) 2024-12-09T16:30:44,935 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bb9e94e1a90f12ab562e74f9c6ecaf2d, disabling compactions & flushes 2024-12-09T16:30:44,935 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733761844859.bb9e94e1a90f12ab562e74f9c6ecaf2d. 2024-12-09T16:30:44,935 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733761844859.bb9e94e1a90f12ab562e74f9c6ecaf2d. 2024-12-09T16:30:44,935 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733761844859.bb9e94e1a90f12ab562e74f9c6ecaf2d. after waiting 0 ms 2024-12-09T16:30:44,935 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733761844859.bb9e94e1a90f12ab562e74f9c6ecaf2d. 2024-12-09T16:30:44,936 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733761844859.bb9e94e1a90f12ab562e74f9c6ecaf2d. 2024-12-09T16:30:44,936 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bb9e94e1a90f12ab562e74f9c6ecaf2d: Waiting for close lock at 1733761844935Disabling compacts and flushes for region at 1733761844935Disabling writes for close at 1733761844935Writing region close event to WAL at 1733761844936 (+1 ms)Closed at 1733761844936 2024-12-09T16:30:44,941 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-81573533/hregion-81573533.1733761844868 not finished, retry = 0 2024-12-09T16:30:44,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741853_1029 (size=93) 2024-12-09T16:30:44,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741853_1029 (size=93) 2024-12-09T16:30:44,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741853_1029 (size=93) 2024-12-09T16:30:45,046 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T16:30:45,046 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-81573533:(num 1733761844868) 2024-12-09T16:30:45,046 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T16:30:45,050 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733761844856, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:30:45,069 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733761844856/wal.1733761845050, exclude list is [], retry=0 2024-12-09T16:30:45,073 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:30:45,074 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:30:45,074 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:30:45,077 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733761844856/wal.1733761845050 2024-12-09T16:30:45,078 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:39661:39661)] 2024-12-09T16:30:45,353 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733761844856/wal.1733761845050, size=0 (0bytes) 2024-12-09T16:30:45,353 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733761844856/wal.1733761845050 might be still open, length is 0 2024-12-09T16:30:45,353 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733761844856/wal.1733761845050 2024-12-09T16:30:45,355 WARN [IPC Server handler 0 on default port 39607 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733761844856/wal.1733761845050 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-09T16:30:45,356 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733761844856/wal.1733761845050 after 3ms 2024-12-09T16:30:47,933 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:53496 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:39721:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53496 dst: /127.0.0.1:39721 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:30:47,933 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:51358 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41451:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51358 dst: /127.0.0.1:41451 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:30:47,932 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:37354 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:36895:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37354 dst: /127.0.0.1:36895 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36895 remote=/127.0.0.1:37354]. Total timeout mills is 60000, 57374 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:30:47,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741855_1032 (size=200589) 2024-12-09T16:30:47,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741855_1032 (size=200589) 2024-12-09T16:30:47,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741855_1032 (size=200589) 2024-12-09T16:30:48,765 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T16:30:48,841 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-09T16:30:49,357 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733761844856/wal.1733761845050 after 4004ms 2024-12-09T16:30:49,361 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733761844856/wal.1733761845050: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:30:49,363 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733761844856/wal.1733761845050 took 4011ms 2024-12-09T16:30:49,368 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733761845050.temp 2024-12-09T16:30:49,378 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/recovered.edits/0000000000000000001-wal.1733761845050.temp 2024-12-09T16:30:49,539 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733761844856/wal.1733761845050; continuing. 2024-12-09T16:30:49,539 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733761844856/wal.1733761845050 so closing down 2024-12-09T16:30:49,539 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T16:30:49,541 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T16:30:49,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741856_1033 (size=200597) 2024-12-09T16:30:49,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741856_1033 (size=200597) 2024-12-09T16:30:49,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741856_1033 (size=200597) 2024-12-09T16:30:49,559 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/recovered.edits/0000000000000000001-wal.1733761845050.temp (wrote 3002 edits, skipped 0 edits in 52 ms) 2024-12-09T16:30:49,564 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/recovered.edits/0000000000000000001-wal.1733761845050.temp to hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/recovered.edits/0000000000000003002 2024-12-09T16:30:49,566 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 202 ms; skipped=0; WAL=hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733761844856/wal.1733761845050, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T16:30:49,566 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733761844856/wal.1733761845050, journal: Splitting hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733761844856/wal.1733761845050, size=0 (0bytes) at 1733761845353Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/recovered.edits/0000000000000000001-wal.1733761845050.temp at 1733761849378 (+4025 ms)Split 1024 edits, skipped 0 edits. at 1733761849463 (+85 ms)Split 2048 edits, skipped 0 edits. at 1733761849505 (+42 ms)Finishing writing output for hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733761844856/wal.1733761845050 so closing down at 1733761849539 (+34 ms)3 split writer threads finished at 1733761849541 (+2 ms)Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/recovered.edits/0000000000000000001-wal.1733761845050.temp (wrote 3002 edits, skipped 0 edits in 52 ms) at 1733761849560 (+19 ms)Rename recovered edits hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/recovered.edits/0000000000000000001-wal.1733761845050.temp to hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/recovered.edits/0000000000000003002 at 1733761849564 (+4 ms)Processed 3002 edits across 1 Regions in 202 ms; skipped=0; WAL=hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733761844856/wal.1733761845050, size=0, length=0, corrupted=false, cancelled=false at 1733761849566 (+2 ms) 2024-12-09T16:30:49,570 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733761844856/wal.1733761845050 to hdfs://localhost:39607/hbase/oldWALs/wal.1733761845050 2024-12-09T16:30:49,574 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/recovered.edits/0000000000000003002 2024-12-09T16:30:49,574 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T16:30:49,581 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733761844856, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:30:49,600 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733761844856/wal.1733761849581, exclude list is [], retry=0 2024-12-09T16:30:49,607 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:30:49,608 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:30:49,609 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:30:49,622 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733761844856/wal.1733761849581 2024-12-09T16:30:49,624 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:39661:39661)] 2024-12-09T16:30:49,625 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733761844859.bb9e94e1a90f12ab562e74f9c6ecaf2d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:30:49,628 INFO [StoreOpener-bb9e94e1a90f12ab562e74f9c6ecaf2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bb9e94e1a90f12ab562e74f9c6ecaf2d 2024-12-09T16:30:49,630 INFO [StoreOpener-bb9e94e1a90f12ab562e74f9c6ecaf2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb9e94e1a90f12ab562e74f9c6ecaf2d columnFamilyName a 2024-12-09T16:30:49,630 DEBUG [StoreOpener-bb9e94e1a90f12ab562e74f9c6ecaf2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:49,631 INFO [StoreOpener-bb9e94e1a90f12ab562e74f9c6ecaf2d-1 {}] regionserver.HStore(327): Store=bb9e94e1a90f12ab562e74f9c6ecaf2d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:49,631 INFO [StoreOpener-bb9e94e1a90f12ab562e74f9c6ecaf2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bb9e94e1a90f12ab562e74f9c6ecaf2d 2024-12-09T16:30:49,633 INFO [StoreOpener-bb9e94e1a90f12ab562e74f9c6ecaf2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb9e94e1a90f12ab562e74f9c6ecaf2d columnFamilyName b 2024-12-09T16:30:49,633 DEBUG [StoreOpener-bb9e94e1a90f12ab562e74f9c6ecaf2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:49,634 INFO [StoreOpener-bb9e94e1a90f12ab562e74f9c6ecaf2d-1 {}] regionserver.HStore(327): Store=bb9e94e1a90f12ab562e74f9c6ecaf2d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:49,634 INFO [StoreOpener-bb9e94e1a90f12ab562e74f9c6ecaf2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bb9e94e1a90f12ab562e74f9c6ecaf2d 2024-12-09T16:30:49,636 INFO [StoreOpener-bb9e94e1a90f12ab562e74f9c6ecaf2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb9e94e1a90f12ab562e74f9c6ecaf2d columnFamilyName c 2024-12-09T16:30:49,636 DEBUG [StoreOpener-bb9e94e1a90f12ab562e74f9c6ecaf2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:49,637 INFO [StoreOpener-bb9e94e1a90f12ab562e74f9c6ecaf2d-1 {}] regionserver.HStore(327): Store=bb9e94e1a90f12ab562e74f9c6ecaf2d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:49,638 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bb9e94e1a90f12ab562e74f9c6ecaf2d 2024-12-09T16:30:49,639 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d 2024-12-09T16:30:49,642 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d 2024-12-09T16:30:49,643 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/recovered.edits/0000000000000003002 2024-12-09T16:30:49,648 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:30:49,701 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T16:30:50,137 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bb9e94e1a90f12ab562e74f9c6ecaf2d 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-09T16:30:50,173 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/.tmp/a/a5034c8f1d60408ca981dc1060adf183 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733761845090/Put/seqid=0 2024-12-09T16:30:50,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741858_1035 (size=50463) 2024-12-09T16:30:50,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741858_1035 (size=50463) 2024-12-09T16:30:50,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741858_1035 (size=50463) 2024-12-09T16:30:50,194 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/.tmp/a/a5034c8f1d60408ca981dc1060adf183 2024-12-09T16:30:50,204 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/.tmp/a/a5034c8f1d60408ca981dc1060adf183 as hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/a/a5034c8f1d60408ca981dc1060adf183 2024-12-09T16:30:50,213 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/a/a5034c8f1d60408ca981dc1060adf183, entries=754, sequenceid=754, filesize=49.3 K 2024-12-09T16:30:50,214 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for bb9e94e1a90f12ab562e74f9c6ecaf2d in 77ms, sequenceid=754, compaction requested=false; wal=null 2024-12-09T16:30:50,234 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-09T16:30:50,235 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-09T16:30:50,237 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T16:30:50,237 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-09T16:30:50,238 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-09T16:30:50,238 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-09T16:30:50,239 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-09T16:30:50,239 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-09T16:30:50,246 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T16:30:50,246 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bb9e94e1a90f12ab562e74f9c6ecaf2d 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-09T16:30:50,260 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/.tmp/a/7792381113ce4e1ca55601133826daa6 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733761845153/Put/seqid=0 2024-12-09T16:30:50,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741859_1036 (size=20072) 2024-12-09T16:30:50,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741859_1036 (size=20072) 2024-12-09T16:30:50,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741859_1036 (size=20072) 2024-12-09T16:30:50,278 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/.tmp/a/7792381113ce4e1ca55601133826daa6 2024-12-09T16:30:50,315 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/.tmp/b/21ccaa54550045ba843a01968127aeb1 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733761845180/Put/seqid=0 2024-12-09T16:30:50,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741860_1037 (size=35835) 2024-12-09T16:30:50,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741860_1037 (size=35835) 2024-12-09T16:30:50,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741860_1037 (size=35835) 2024-12-09T16:30:50,335 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/.tmp/b/21ccaa54550045ba843a01968127aeb1 2024-12-09T16:30:50,345 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/.tmp/a/7792381113ce4e1ca55601133826daa6 as hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/a/7792381113ce4e1ca55601133826daa6 2024-12-09T16:30:50,375 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/a/7792381113ce4e1ca55601133826daa6, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-09T16:30:50,377 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/.tmp/b/21ccaa54550045ba843a01968127aeb1 as hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/b/21ccaa54550045ba843a01968127aeb1 2024-12-09T16:30:50,404 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/b/21ccaa54550045ba843a01968127aeb1, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-09T16:30:50,404 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for bb9e94e1a90f12ab562e74f9c6ecaf2d in 158ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-09T16:30:50,419 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T16:30:50,420 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bb9e94e1a90f12ab562e74f9c6ecaf2d 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-09T16:30:50,428 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/.tmp/b/b5de61b684fd47949e4e7413c160980f is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733761845208/Put/seqid=0 2024-12-09T16:30:50,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741861_1038 (size=35082) 2024-12-09T16:30:50,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741861_1038 (size=35082) 2024-12-09T16:30:50,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741861_1038 (size=35082) 2024-12-09T16:30:50,442 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/.tmp/b/b5de61b684fd47949e4e7413c160980f 2024-12-09T16:30:50,468 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/.tmp/c/720502f989b34d0080d36ba745f54863 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733761845251/Put/seqid=0 2024-12-09T16:30:50,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741862_1039 (size=20825) 2024-12-09T16:30:50,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741862_1039 (size=20825) 2024-12-09T16:30:50,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741862_1039 (size=20825) 2024-12-09T16:30:50,483 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/.tmp/c/720502f989b34d0080d36ba745f54863 2024-12-09T16:30:50,493 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/.tmp/b/b5de61b684fd47949e4e7413c160980f as hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/b/b5de61b684fd47949e4e7413c160980f 2024-12-09T16:30:50,510 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/b/b5de61b684fd47949e4e7413c160980f, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-09T16:30:50,514 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/.tmp/c/720502f989b34d0080d36ba745f54863 as hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/c/720502f989b34d0080d36ba745f54863 2024-12-09T16:30:50,523 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/c/720502f989b34d0080d36ba745f54863, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-09T16:30:50,524 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for bb9e94e1a90f12ab562e74f9c6ecaf2d in 104ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-09T16:30:50,538 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733761845304/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733761844859.bb9e94e1a90f12ab562e74f9c6ecaf2d. 2024-12-09T16:30:50,542 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/recovered.edits/0000000000000003002 2024-12-09T16:30:50,542 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T16:30:50,543 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bb9e94e1a90f12ab562e74f9c6ecaf2d 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-09T16:30:50,555 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/.tmp/c/31a182a38dca42f18ce08eb35c4b9d06 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733761845261/Put/seqid=0 2024-12-09T16:30:50,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741863_1040 (size=50301) 2024-12-09T16:30:50,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741863_1040 (size=50301) 2024-12-09T16:30:50,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741863_1040 (size=50301) 2024-12-09T16:30:50,571 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/.tmp/c/31a182a38dca42f18ce08eb35c4b9d06 2024-12-09T16:30:50,578 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 31a182a38dca42f18ce08eb35c4b9d06 2024-12-09T16:30:50,580 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/.tmp/c/31a182a38dca42f18ce08eb35c4b9d06 as hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/c/31a182a38dca42f18ce08eb35c4b9d06 2024-12-09T16:30:50,587 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 31a182a38dca42f18ce08eb35c4b9d06 2024-12-09T16:30:50,587 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/c/31a182a38dca42f18ce08eb35c4b9d06, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-09T16:30:50,588 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for bb9e94e1a90f12ab562e74f9c6ecaf2d in 45ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-09T16:30:50,589 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/recovered.edits/0000000000000003002 2024-12-09T16:30:50,591 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bb9e94e1a90f12ab562e74f9c6ecaf2d 2024-12-09T16:30:50,591 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bb9e94e1a90f12ab562e74f9c6ecaf2d 2024-12-09T16:30:50,592 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T16:30:50,596 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bb9e94e1a90f12ab562e74f9c6ecaf2d 2024-12-09T16:30:50,601 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenIntoWAL/bb9e94e1a90f12ab562e74f9c6ecaf2d/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-09T16:30:50,603 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bb9e94e1a90f12ab562e74f9c6ecaf2d; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69470070, jitterRate=0.03518471121788025}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T16:30:50,603 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bb9e94e1a90f12ab562e74f9c6ecaf2d: Writing region info on filesystem at 1733761849625Initializing all the Stores at 1733761849627 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761849627Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761849627Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761849627Cleaning up temporary data from old regions at 1733761850591 (+964 ms)Region opened successfully at 1733761850603 (+12 ms) 2024-12-09T16:30:50,679 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bb9e94e1a90f12ab562e74f9c6ecaf2d, disabling compactions & flushes 2024-12-09T16:30:50,679 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733761844859.bb9e94e1a90f12ab562e74f9c6ecaf2d. 2024-12-09T16:30:50,679 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733761844859.bb9e94e1a90f12ab562e74f9c6ecaf2d. 2024-12-09T16:30:50,679 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733761844859.bb9e94e1a90f12ab562e74f9c6ecaf2d. after waiting 0 ms 2024-12-09T16:30:50,679 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733761844859.bb9e94e1a90f12ab562e74f9c6ecaf2d. 2024-12-09T16:30:50,686 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733761844859.bb9e94e1a90f12ab562e74f9c6ecaf2d. 2024-12-09T16:30:50,686 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bb9e94e1a90f12ab562e74f9c6ecaf2d: Waiting for close lock at 1733761850679Disabling compacts and flushes for region at 1733761850679Disabling writes for close at 1733761850679Writing region close event to WAL at 1733761850686 (+7 ms)Closed at 1733761850686 2024-12-09T16:30:50,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741857_1034 (size=93) 2024-12-09T16:30:50,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741857_1034 (size=93) 2024-12-09T16:30:50,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741857_1034 (size=93) 2024-12-09T16:30:50,695 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T16:30:50,695 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733761849581) 2024-12-09T16:30:50,717 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=403 (was 389) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545611582_22 at /127.0.0.1:53546 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545611582_22 at /127.0.0.1:37396 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545611582_22 at /127.0.0.1:51370 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:36531 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40409 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:39607 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-545611582_22 at /127.0.0.1:37412 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36531 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:40409 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/cluster_cd98e670-d246-b6f3-ff89-5ca1891baf9f/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/cluster_cd98e670-d246-b6f3-ff89-5ca1891baf9f/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:39607 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=662 (was 703), ProcessCount=11 (was 11), AvailableMemoryMB=6735 (was 6841) 2024-12-09T16:30:50,742 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=403, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=662, ProcessCount=11, AvailableMemoryMB=6735 2024-12-09T16:30:50,766 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:30:50,770 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:30:50,771 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T16:30:50,776 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-95407521, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/hregion-95407521, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:30:50,795 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-95407521/hregion-95407521.1733761850776, exclude list is [], retry=0 2024-12-09T16:30:50,803 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:30:50,804 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:30:50,805 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:30:50,816 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-95407521/hregion-95407521.1733761850776 2024-12-09T16:30:50,818 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:39661:39661)] 2024-12-09T16:30:50,818 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 101e30924bf886b1d156cd48ef5eca27, NAME => 'test2727,,1733761850767.101e30924bf886b1d156cd48ef5eca27.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39607/hbase 2024-12-09T16:30:50,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741865_1042 (size=43) 2024-12-09T16:30:50,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741865_1042 (size=43) 2024-12-09T16:30:50,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741865_1042 (size=43) 2024-12-09T16:30:50,846 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733761850767.101e30924bf886b1d156cd48ef5eca27.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:30:50,849 INFO [StoreOpener-101e30924bf886b1d156cd48ef5eca27-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 101e30924bf886b1d156cd48ef5eca27 2024-12-09T16:30:50,851 INFO [StoreOpener-101e30924bf886b1d156cd48ef5eca27-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 101e30924bf886b1d156cd48ef5eca27 columnFamilyName a 2024-12-09T16:30:50,851 DEBUG [StoreOpener-101e30924bf886b1d156cd48ef5eca27-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:50,852 INFO [StoreOpener-101e30924bf886b1d156cd48ef5eca27-1 {}] regionserver.HStore(327): Store=101e30924bf886b1d156cd48ef5eca27/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:50,852 INFO [StoreOpener-101e30924bf886b1d156cd48ef5eca27-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 101e30924bf886b1d156cd48ef5eca27 2024-12-09T16:30:50,855 INFO [StoreOpener-101e30924bf886b1d156cd48ef5eca27-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 101e30924bf886b1d156cd48ef5eca27 columnFamilyName b 2024-12-09T16:30:50,855 DEBUG [StoreOpener-101e30924bf886b1d156cd48ef5eca27-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:50,856 INFO [StoreOpener-101e30924bf886b1d156cd48ef5eca27-1 {}] regionserver.HStore(327): Store=101e30924bf886b1d156cd48ef5eca27/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:50,856 INFO [StoreOpener-101e30924bf886b1d156cd48ef5eca27-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 101e30924bf886b1d156cd48ef5eca27 2024-12-09T16:30:50,858 INFO [StoreOpener-101e30924bf886b1d156cd48ef5eca27-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 101e30924bf886b1d156cd48ef5eca27 columnFamilyName c 2024-12-09T16:30:50,858 DEBUG [StoreOpener-101e30924bf886b1d156cd48ef5eca27-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:50,859 INFO [StoreOpener-101e30924bf886b1d156cd48ef5eca27-1 {}] regionserver.HStore(327): Store=101e30924bf886b1d156cd48ef5eca27/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:50,859 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 101e30924bf886b1d156cd48ef5eca27 2024-12-09T16:30:50,860 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27 2024-12-09T16:30:50,861 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27 2024-12-09T16:30:50,863 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 101e30924bf886b1d156cd48ef5eca27 2024-12-09T16:30:50,863 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 101e30924bf886b1d156cd48ef5eca27 2024-12-09T16:30:50,863 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T16:30:50,865 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 101e30924bf886b1d156cd48ef5eca27 2024-12-09T16:30:50,868 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T16:30:50,869 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 101e30924bf886b1d156cd48ef5eca27; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65418547, jitterRate=-0.02518768608570099}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T16:30:50,870 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 101e30924bf886b1d156cd48ef5eca27: Writing region info on filesystem at 1733761850846Initializing all the Stores at 1733761850848 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761850848Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761850848Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761850848Cleaning up temporary data from old regions at 1733761850863 (+15 ms)Region opened successfully at 1733761850870 (+7 ms) 2024-12-09T16:30:50,870 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 101e30924bf886b1d156cd48ef5eca27, disabling compactions & flushes 2024-12-09T16:30:50,870 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733761850767.101e30924bf886b1d156cd48ef5eca27. 2024-12-09T16:30:50,870 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733761850767.101e30924bf886b1d156cd48ef5eca27. 2024-12-09T16:30:50,871 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733761850767.101e30924bf886b1d156cd48ef5eca27. after waiting 0 ms 2024-12-09T16:30:50,871 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733761850767.101e30924bf886b1d156cd48ef5eca27. 2024-12-09T16:30:50,871 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733761850767.101e30924bf886b1d156cd48ef5eca27. 2024-12-09T16:30:50,871 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 101e30924bf886b1d156cd48ef5eca27: Waiting for close lock at 1733761850870Disabling compacts and flushes for region at 1733761850870Disabling writes for close at 1733761850871 (+1 ms)Writing region close event to WAL at 1733761850871Closed at 1733761850871 2024-12-09T16:30:50,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741864_1041 (size=93) 2024-12-09T16:30:50,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741864_1041 (size=93) 2024-12-09T16:30:50,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741864_1041 (size=93) 2024-12-09T16:30:50,880 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T16:30:50,880 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-95407521:(num 1733761850776) 2024-12-09T16:30:50,880 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T16:30:50,883 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:30:50,905 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761850884, exclude list is [], retry=0 2024-12-09T16:30:50,909 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:30:50,910 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:30:50,911 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:30:50,913 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761850884 2024-12-09T16:30:50,914 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39661:39661),(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:35519:35519)] 2024-12-09T16:30:51,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741866_1043 (size=200357) 2024-12-09T16:30:51,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741866_1043 (size=200357) 2024-12-09T16:30:51,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741866_1043 (size=200357) 2024-12-09T16:30:51,141 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761850884, size=195.7 K (200357bytes) 2024-12-09T16:30:51,141 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761850884 2024-12-09T16:30:51,142 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761850884 after 1ms 2024-12-09T16:30:51,145 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761850884: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:30:51,147 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761850884 took 6ms 2024-12-09T16:30:51,157 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733761850884.temp 2024-12-09T16:30:51,162 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000000001-wal.1733761850884.temp 2024-12-09T16:30:51,230 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761850884 so closing down 2024-12-09T16:30:51,230 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T16:30:51,230 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T16:30:51,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741867_1044 (size=200357) 2024-12-09T16:30:51,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741867_1044 (size=200357) 2024-12-09T16:30:51,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741867_1044 (size=200357) 2024-12-09T16:30:51,241 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000000001-wal.1733761850884.temp (wrote 3000 edits, skipped 0 edits in 20 ms) 2024-12-09T16:30:51,244 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000000001-wal.1733761850884.temp to hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000003000 2024-12-09T16:30:51,244 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 94 ms; skipped=0; WAL=hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761850884, size=195.7 K, length=200357, corrupted=false, cancelled=false 2024-12-09T16:30:51,244 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761850884, journal: Splitting hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761850884, size=195.7 K (200357bytes) at 1733761851141Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000000001-wal.1733761850884.temp at 1733761851162 (+21 ms)Split 1024 edits, skipped 0 edits. at 1733761851172 (+10 ms)Split 2048 edits, skipped 0 edits. at 1733761851195 (+23 ms)Finishing writing output for hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761850884 so closing down at 1733761851230 (+35 ms)3 split writer threads finished at 1733761851230Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000000001-wal.1733761850884.temp (wrote 3000 edits, skipped 0 edits in 20 ms) at 1733761851241 (+11 ms)Rename recovered edits hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000000001-wal.1733761850884.temp to hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000003000 at 1733761851244 (+3 ms)Processed 3000 edits across 1 Regions in 94 ms; skipped=0; WAL=hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761850884, size=195.7 K, length=200357, corrupted=false, cancelled=false at 1733761851244 2024-12-09T16:30:51,248 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761850884 to hdfs://localhost:39607/hbase/oldWALs/wal.1733761850884 2024-12-09T16:30:51,250 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000003000 2024-12-09T16:30:51,251 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T16:30:51,255 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:30:51,278 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761851256, exclude list is [], retry=0 2024-12-09T16:30:51,283 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:30:51,283 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:30:51,284 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:30:51,288 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761851256 2024-12-09T16:30:51,288 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:39661:39661)] 2024-12-09T16:30:51,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741868_1045 (size=200484) 2024-12-09T16:30:51,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741868_1045 (size=200484) 2024-12-09T16:30:51,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741868_1045 (size=200484) 2024-12-09T16:30:51,540 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761851256, size=195.8 K (200484bytes) 2024-12-09T16:30:51,540 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761851256 2024-12-09T16:30:51,541 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761851256 after 1ms 2024-12-09T16:30:51,544 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761851256: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:30:51,547 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761851256 took 7ms 2024-12-09T16:30:51,555 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733761851256.temp 2024-12-09T16:30:51,562 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000003001-wal.1733761851256.temp 2024-12-09T16:30:51,601 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761851256 so closing down 2024-12-09T16:30:51,601 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T16:30:51,604 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T16:30:51,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741869_1046 (size=200484) 2024-12-09T16:30:51,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741869_1046 (size=200484) 2024-12-09T16:30:51,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741869_1046 (size=200484) 2024-12-09T16:30:51,615 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000003001-wal.1733761851256.temp (wrote 3000 edits, skipped 0 edits in 19 ms) 2024-12-09T16:30:51,617 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000003001-wal.1733761851256.temp to hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000006000 2024-12-09T16:30:51,618 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 70 ms; skipped=0; WAL=hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761851256, size=195.8 K, length=200484, corrupted=false, cancelled=false 2024-12-09T16:30:51,618 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761851256, journal: Splitting hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761851256, size=195.8 K (200484bytes) at 1733761851540Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000003001-wal.1733761851256.temp at 1733761851562 (+22 ms)Split 1024 edits, skipped 0 edits. at 1733761851569 (+7 ms)Split 2048 edits, skipped 0 edits. at 1733761851587 (+18 ms)Finishing writing output for hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761851256 so closing down at 1733761851601 (+14 ms)3 split writer threads finished at 1733761851604 (+3 ms)Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000003001-wal.1733761851256.temp (wrote 3000 edits, skipped 0 edits in 19 ms) at 1733761851615 (+11 ms)Rename recovered edits hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000003001-wal.1733761851256.temp to hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000006000 at 1733761851617 (+2 ms)Processed 3000 edits across 1 Regions in 70 ms; skipped=0; WAL=hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761851256, size=195.8 K, length=200484, corrupted=false, cancelled=false at 1733761851618 (+1 ms) 2024-12-09T16:30:51,629 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761851256 to hdfs://localhost:39607/hbase/oldWALs/wal.1733761851256 2024-12-09T16:30:51,637 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000006000 2024-12-09T16:30:51,638 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T16:30:51,641 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/test2727-manual,16010,1733761850765, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:30:51,664 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761851642, exclude list is [], retry=0 2024-12-09T16:30:51,667 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:30:51,668 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:30:51,669 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:30:51,675 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761851642 2024-12-09T16:30:51,675 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:39661:39661)] 2024-12-09T16:30:51,675 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 101e30924bf886b1d156cd48ef5eca27, NAME => 'test2727,,1733761850767.101e30924bf886b1d156cd48ef5eca27.', STARTKEY => '', ENDKEY => ''} 2024-12-09T16:30:51,676 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733761850767.101e30924bf886b1d156cd48ef5eca27.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:30:51,676 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 101e30924bf886b1d156cd48ef5eca27 2024-12-09T16:30:51,676 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 101e30924bf886b1d156cd48ef5eca27 2024-12-09T16:30:51,681 INFO [StoreOpener-101e30924bf886b1d156cd48ef5eca27-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 101e30924bf886b1d156cd48ef5eca27 2024-12-09T16:30:51,683 INFO [StoreOpener-101e30924bf886b1d156cd48ef5eca27-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 101e30924bf886b1d156cd48ef5eca27 columnFamilyName a 2024-12-09T16:30:51,683 DEBUG [StoreOpener-101e30924bf886b1d156cd48ef5eca27-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:51,684 INFO [StoreOpener-101e30924bf886b1d156cd48ef5eca27-1 {}] regionserver.HStore(327): Store=101e30924bf886b1d156cd48ef5eca27/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:51,684 INFO [StoreOpener-101e30924bf886b1d156cd48ef5eca27-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 101e30924bf886b1d156cd48ef5eca27 2024-12-09T16:30:51,686 INFO [StoreOpener-101e30924bf886b1d156cd48ef5eca27-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 101e30924bf886b1d156cd48ef5eca27 columnFamilyName b 2024-12-09T16:30:51,686 DEBUG [StoreOpener-101e30924bf886b1d156cd48ef5eca27-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:51,687 INFO [StoreOpener-101e30924bf886b1d156cd48ef5eca27-1 {}] regionserver.HStore(327): Store=101e30924bf886b1d156cd48ef5eca27/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:51,687 INFO [StoreOpener-101e30924bf886b1d156cd48ef5eca27-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 101e30924bf886b1d156cd48ef5eca27 2024-12-09T16:30:51,689 INFO [StoreOpener-101e30924bf886b1d156cd48ef5eca27-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 101e30924bf886b1d156cd48ef5eca27 columnFamilyName c 2024-12-09T16:30:51,689 DEBUG [StoreOpener-101e30924bf886b1d156cd48ef5eca27-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:51,690 INFO [StoreOpener-101e30924bf886b1d156cd48ef5eca27-1 {}] regionserver.HStore(327): Store=101e30924bf886b1d156cd48ef5eca27/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:51,691 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 101e30924bf886b1d156cd48ef5eca27 2024-12-09T16:30:51,692 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27 2024-12-09T16:30:51,696 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27 2024-12-09T16:30:51,697 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000003000 2024-12-09T16:30:51,701 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:30:51,757 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000003000 2024-12-09T16:30:51,760 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000006000 2024-12-09T16:30:51,765 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:30:51,828 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000006000 2024-12-09T16:30:51,829 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 101e30924bf886b1d156cd48ef5eca27 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-09T16:30:51,860 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/.tmp/a/265d40068e0b4ef88d12499c1c7cdaca is 41, key is test2727/a:100/1733761851297/Put/seqid=0 2024-12-09T16:30:51,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741871_1048 (size=84227) 2024-12-09T16:30:51,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741871_1048 (size=84227) 2024-12-09T16:30:51,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741871_1048 (size=84227) 2024-12-09T16:30:51,879 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/.tmp/a/265d40068e0b4ef88d12499c1c7cdaca 2024-12-09T16:30:51,926 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/.tmp/b/b792e28e2cd6413ca0b1934e98c9373f is 41, key is test2727/b:100/1733761851364/Put/seqid=0 2024-12-09T16:30:51,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741872_1049 (size=84609) 2024-12-09T16:30:51,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741872_1049 (size=84609) 2024-12-09T16:30:51,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741872_1049 (size=84609) 2024-12-09T16:30:51,957 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/.tmp/b/b792e28e2cd6413ca0b1934e98c9373f 2024-12-09T16:30:51,990 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/.tmp/c/9c4b95766f234a58a524bcb75bab9430 is 41, key is test2727/c:100/1733761851436/Put/seqid=0 2024-12-09T16:30:52,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741873_1050 (size=84609) 2024-12-09T16:30:52,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741873_1050 (size=84609) 2024-12-09T16:30:52,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741873_1050 (size=84609) 2024-12-09T16:30:52,011 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/.tmp/c/9c4b95766f234a58a524bcb75bab9430 2024-12-09T16:30:52,022 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/.tmp/a/265d40068e0b4ef88d12499c1c7cdaca as hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/a/265d40068e0b4ef88d12499c1c7cdaca 2024-12-09T16:30:52,037 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/a/265d40068e0b4ef88d12499c1c7cdaca, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-09T16:30:52,039 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/.tmp/b/b792e28e2cd6413ca0b1934e98c9373f as hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/b/b792e28e2cd6413ca0b1934e98c9373f 2024-12-09T16:30:52,051 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/b/b792e28e2cd6413ca0b1934e98c9373f, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-09T16:30:52,053 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/.tmp/c/9c4b95766f234a58a524bcb75bab9430 as hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/c/9c4b95766f234a58a524bcb75bab9430 2024-12-09T16:30:52,066 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/c/9c4b95766f234a58a524bcb75bab9430, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-09T16:30:52,066 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 101e30924bf886b1d156cd48ef5eca27 in 237ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-09T16:30:52,070 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000003000 2024-12-09T16:30:52,071 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/0000000000000006000 2024-12-09T16:30:52,073 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 101e30924bf886b1d156cd48ef5eca27 2024-12-09T16:30:52,073 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 101e30924bf886b1d156cd48ef5eca27 2024-12-09T16:30:52,076 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T16:30:52,079 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 101e30924bf886b1d156cd48ef5eca27 2024-12-09T16:30:52,094 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/hbase/data/default/test2727/101e30924bf886b1d156cd48ef5eca27/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-09T16:30:52,099 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 101e30924bf886b1d156cd48ef5eca27; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75125266, jitterRate=0.11945369839668274}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T16:30:52,100 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 101e30924bf886b1d156cd48ef5eca27: Writing region info on filesystem at 1733761851676Initializing all the Stores at 1733761851678 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761851678Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761851681 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761851681Obtaining lock to block concurrent updates at 1733761851829 (+148 ms)Preparing flush snapshotting stores in 101e30924bf886b1d156cd48ef5eca27 at 1733761851829Finished memstore snapshotting test2727,,1733761850767.101e30924bf886b1d156cd48ef5eca27., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733761851829Flushing stores of test2727,,1733761850767.101e30924bf886b1d156cd48ef5eca27. at 1733761851829Flushing 101e30924bf886b1d156cd48ef5eca27/a: creating writer at 1733761851829Flushing 101e30924bf886b1d156cd48ef5eca27/a: appending metadata at 1733761851859 (+30 ms)Flushing 101e30924bf886b1d156cd48ef5eca27/a: closing flushed file at 1733761851859Flushing 101e30924bf886b1d156cd48ef5eca27/b: creating writer at 1733761851892 (+33 ms)Flushing 101e30924bf886b1d156cd48ef5eca27/b: appending metadata at 1733761851922 (+30 ms)Flushing 101e30924bf886b1d156cd48ef5eca27/b: closing flushed file at 1733761851922Flushing 101e30924bf886b1d156cd48ef5eca27/c: creating writer at 1733761851964 (+42 ms)Flushing 101e30924bf886b1d156cd48ef5eca27/c: appending metadata at 1733761851989 (+25 ms)Flushing 101e30924bf886b1d156cd48ef5eca27/c: closing flushed file at 1733761851990 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ae9a062: reopening flushed file at 1733761852021 (+31 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ee27d7e: reopening flushed file at 1733761852038 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e85f418: reopening flushed file at 1733761852051 (+13 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 101e30924bf886b1d156cd48ef5eca27 in 237ms, sequenceid=6000, compaction requested=false; wal=null at 1733761852066 (+15 ms)Cleaning up temporary data from old regions at 1733761852073 (+7 ms)Region opened successfully at 1733761852100 (+27 ms) 2024-12-09T16:30:52,102 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-09T16:30:52,102 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 101e30924bf886b1d156cd48ef5eca27, disabling compactions & flushes 2024-12-09T16:30:52,102 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733761850767.101e30924bf886b1d156cd48ef5eca27. 2024-12-09T16:30:52,102 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733761850767.101e30924bf886b1d156cd48ef5eca27. 2024-12-09T16:30:52,102 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733761850767.101e30924bf886b1d156cd48ef5eca27. after waiting 0 ms 2024-12-09T16:30:52,102 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733761850767.101e30924bf886b1d156cd48ef5eca27. 2024-12-09T16:30:52,108 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733761850767.101e30924bf886b1d156cd48ef5eca27. 2024-12-09T16:30:52,108 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 101e30924bf886b1d156cd48ef5eca27: Waiting for close lock at 1733761852102Disabling compacts and flushes for region at 1733761852102Disabling writes for close at 1733761852102Writing region close event to WAL at 1733761852108 (+6 ms)Closed at 1733761852108 2024-12-09T16:30:52,117 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1733761850765/wal.1733761851642 not finished, retry = 0 2024-12-09T16:30:52,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741870_1047 (size=93) 2024-12-09T16:30:52,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741870_1047 (size=93) 2024-12-09T16:30:52,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741870_1047 (size=93) 2024-12-09T16:30:52,221 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T16:30:52,221 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733761851642) 2024-12-09T16:30:52,236 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=407 (was 403) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:37396 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/cluster_cd98e670-d246-b6f3-ff89-5ca1891baf9f/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:51370 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/cluster_cd98e670-d246-b6f3-ff89-5ca1891baf9f/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:53692 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=665 (was 662) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6513 (was 6735) 2024-12-09T16:30:52,249 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=407, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=665, ProcessCount=11, AvailableMemoryMB=6508 2024-12-09T16:30:52,270 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:30:52,278 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733761852269, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:30:52,279 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733761852279 2024-12-09T16:30:52,306 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733761852269/wal.1733761852279 2024-12-09T16:30:52,309 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:39661:39661)] 2024-12-09T16:30:52,315 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8cdab2fd809a399678859495495531ee, NAME => 'testSequentialEditLogSeqNum,,1733761852271.8cdab2fd809a399678859495495531ee.', STARTKEY => '', ENDKEY => ''} 2024-12-09T16:30:52,315 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733761852271.8cdab2fd809a399678859495495531ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:30:52,316 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8cdab2fd809a399678859495495531ee 2024-12-09T16:30:52,316 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8cdab2fd809a399678859495495531ee 2024-12-09T16:30:52,323 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:39607/hbase/data/default/testSequentialEditLogSeqNum/8cdab2fd809a399678859495495531ee doesn't exist for region: 8cdab2fd809a399678859495495531ee on table testSequentialEditLogSeqNum 2024-12-09T16:30:52,324 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 8cdab2fd809a399678859495495531ee on table testSequentialEditLogSeqNum 2024-12-09T16:30:52,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741875_1052 (size=62) 2024-12-09T16:30:52,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741875_1052 (size=62) 2024-12-09T16:30:52,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741875_1052 (size=62) 2024-12-09T16:30:52,343 INFO [StoreOpener-8cdab2fd809a399678859495495531ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8cdab2fd809a399678859495495531ee 2024-12-09T16:30:52,346 INFO [StoreOpener-8cdab2fd809a399678859495495531ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8cdab2fd809a399678859495495531ee columnFamilyName a 2024-12-09T16:30:52,346 DEBUG [StoreOpener-8cdab2fd809a399678859495495531ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:52,349 INFO [StoreOpener-8cdab2fd809a399678859495495531ee-1 {}] regionserver.HStore(327): Store=8cdab2fd809a399678859495495531ee/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:52,349 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8cdab2fd809a399678859495495531ee 2024-12-09T16:30:52,350 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testSequentialEditLogSeqNum/8cdab2fd809a399678859495495531ee 2024-12-09T16:30:52,351 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testSequentialEditLogSeqNum/8cdab2fd809a399678859495495531ee 2024-12-09T16:30:52,351 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8cdab2fd809a399678859495495531ee 2024-12-09T16:30:52,352 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8cdab2fd809a399678859495495531ee 2024-12-09T16:30:52,354 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8cdab2fd809a399678859495495531ee 2024-12-09T16:30:52,357 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/hbase/data/default/testSequentialEditLogSeqNum/8cdab2fd809a399678859495495531ee/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T16:30:52,358 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8cdab2fd809a399678859495495531ee; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62876920, jitterRate=-0.06306087970733643}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T16:30:52,359 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8cdab2fd809a399678859495495531ee: Writing region info on filesystem at 1733761852316Initializing all the Stores at 1733761852343 (+27 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761852343Cleaning up temporary data from old regions at 1733761852352 (+9 ms)Region opened successfully at 1733761852359 (+7 ms) 2024-12-09T16:30:52,373 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8cdab2fd809a399678859495495531ee 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-09T16:30:52,401 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testSequentialEditLogSeqNum/8cdab2fd809a399678859495495531ee/.tmp/a/5f717dab942b445e9d5b058e9889c1ca is 81, key is testSequentialEditLogSeqNum/a:x0/1733761852359/Put/seqid=0 2024-12-09T16:30:52,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741876_1053 (size=5833) 2024-12-09T16:30:52,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741876_1053 (size=5833) 2024-12-09T16:30:52,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741876_1053 (size=5833) 2024-12-09T16:30:52,421 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/testSequentialEditLogSeqNum/8cdab2fd809a399678859495495531ee/.tmp/a/5f717dab942b445e9d5b058e9889c1ca 2024-12-09T16:30:52,432 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testSequentialEditLogSeqNum/8cdab2fd809a399678859495495531ee/.tmp/a/5f717dab942b445e9d5b058e9889c1ca as hdfs://localhost:39607/hbase/data/default/testSequentialEditLogSeqNum/8cdab2fd809a399678859495495531ee/a/5f717dab942b445e9d5b058e9889c1ca 2024-12-09T16:30:52,442 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/testSequentialEditLogSeqNum/8cdab2fd809a399678859495495531ee/a/5f717dab942b445e9d5b058e9889c1ca, entries=10, sequenceid=13, filesize=5.7 K 2024-12-09T16:30:52,446 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 8cdab2fd809a399678859495495531ee in 73ms, sequenceid=13, compaction requested=false 2024-12-09T16:30:52,446 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8cdab2fd809a399678859495495531ee: 2024-12-09T16:30:52,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T16:30:52,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T16:30:52,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T16:30:52,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T16:30:52,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T16:30:52,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741874_1051 (size=1616) 2024-12-09T16:30:52,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741874_1051 (size=1616) 2024-12-09T16:30:52,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741874_1051 (size=1616) 2024-12-09T16:30:52,491 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39607/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733761852269/wal.1733761852279, size=1.6 K (1616bytes) 2024-12-09T16:30:52,491 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39607/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733761852269/wal.1733761852279 2024-12-09T16:30:52,492 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39607/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733761852269/wal.1733761852279 after 1ms 2024-12-09T16:30:52,498 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733761852269/wal.1733761852279: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:30:52,499 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39607/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733761852269/wal.1733761852279 took 8ms 2024-12-09T16:30:52,510 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39607/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733761852269/wal.1733761852279 so closing down 2024-12-09T16:30:52,510 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T16:30:52,516 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733761852279.temp 2024-12-09T16:30:52,519 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testSequentialEditLogSeqNum/8cdab2fd809a399678859495495531ee/recovered.edits/0000000000000000003-wal.1733761852279.temp 2024-12-09T16:30:52,522 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T16:30:52,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741877_1054 (size=1175) 2024-12-09T16:30:52,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741877_1054 (size=1175) 2024-12-09T16:30:52,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741877_1054 (size=1175) 2024-12-09T16:30:52,594 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testSequentialEditLogSeqNum/8cdab2fd809a399678859495495531ee/recovered.edits/0000000000000000003-wal.1733761852279.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-09T16:30:52,640 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39607/hbase/data/default/testSequentialEditLogSeqNum/8cdab2fd809a399678859495495531ee/recovered.edits/0000000000000000003-wal.1733761852279.temp to hdfs://localhost:39607/hbase/data/default/testSequentialEditLogSeqNum/8cdab2fd809a399678859495495531ee/recovered.edits/0000000000000000020 2024-12-09T16:30:52,640 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 133 ms; skipped=2; WAL=hdfs://localhost:39607/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733761852269/wal.1733761852279, size=1.6 K, length=1616, corrupted=false, cancelled=false 2024-12-09T16:30:52,640 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39607/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733761852269/wal.1733761852279, journal: Splitting hdfs://localhost:39607/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733761852269/wal.1733761852279, size=1.6 K (1616bytes) at 1733761852491Finishing writing output for hdfs://localhost:39607/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733761852269/wal.1733761852279 so closing down at 1733761852510 (+19 ms)Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testSequentialEditLogSeqNum/8cdab2fd809a399678859495495531ee/recovered.edits/0000000000000000003-wal.1733761852279.temp at 1733761852519 (+9 ms)3 split writer threads finished at 1733761852522 (+3 ms)Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testSequentialEditLogSeqNum/8cdab2fd809a399678859495495531ee/recovered.edits/0000000000000000003-wal.1733761852279.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733761852629 (+107 ms)Rename recovered edits hdfs://localhost:39607/hbase/data/default/testSequentialEditLogSeqNum/8cdab2fd809a399678859495495531ee/recovered.edits/0000000000000000003-wal.1733761852279.temp to hdfs://localhost:39607/hbase/data/default/testSequentialEditLogSeqNum/8cdab2fd809a399678859495495531ee/recovered.edits/0000000000000000020 at 1733761852640 (+11 ms)Processed 17 edits across 1 Regions in 133 ms; skipped=2; WAL=hdfs://localhost:39607/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733761852269/wal.1733761852279, size=1.6 K, length=1616, corrupted=false, cancelled=false at 1733761852640 2024-12-09T16:30:52,668 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=408 (was 407) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:37396 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:53692 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=959 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=665 (was 665), ProcessCount=11 (was 11), AvailableMemoryMB=6444 (was 6508) 2024-12-09T16:30:52,687 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=408, OpenFileDescriptor=959, MaxFileDescriptor=1048576, SystemLoadAverage=665, ProcessCount=11, AvailableMemoryMB=6439 2024-12-09T16:30:52,714 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:30:52,717 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:30:52,754 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T16:30:52,767 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-00071097, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/hregion-00071097, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:30:52,782 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-00071097/hregion-00071097.1733761852768, exclude list is [], retry=0 2024-12-09T16:30:52,790 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:30:52,790 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:30:52,791 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:30:52,797 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-00071097/hregion-00071097.1733761852768 2024-12-09T16:30:52,798 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:39661:39661),(127.0.0.1/127.0.0.1:37099:37099)] 2024-12-09T16:30:52,798 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f906899ac9eecdfad6361a762a2e3c3b, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733761852714.f906899ac9eecdfad6361a762a2e3c3b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39607/hbase 2024-12-09T16:30:52,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741879_1056 (size=70) 2024-12-09T16:30:52,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741879_1056 (size=70) 2024-12-09T16:30:52,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741879_1056 (size=70) 2024-12-09T16:30:53,247 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733761852714.f906899ac9eecdfad6361a762a2e3c3b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:30:53,249 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:53,251 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f906899ac9eecdfad6361a762a2e3c3b columnFamilyName a 2024-12-09T16:30:53,251 DEBUG [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:53,252 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] regionserver.HStore(327): Store=f906899ac9eecdfad6361a762a2e3c3b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:53,253 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:53,255 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f906899ac9eecdfad6361a762a2e3c3b columnFamilyName b 2024-12-09T16:30:53,255 DEBUG [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:53,256 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] regionserver.HStore(327): Store=f906899ac9eecdfad6361a762a2e3c3b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:53,256 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:53,258 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f906899ac9eecdfad6361a762a2e3c3b columnFamilyName c 2024-12-09T16:30:53,258 DEBUG [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:53,259 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] regionserver.HStore(327): Store=f906899ac9eecdfad6361a762a2e3c3b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:53,259 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:53,260 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:53,260 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:53,262 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:53,262 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:53,262 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T16:30:53,264 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:53,267 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T16:30:53,267 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f906899ac9eecdfad6361a762a2e3c3b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62780898, jitterRate=-0.06449171900749207}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T16:30:53,268 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f906899ac9eecdfad6361a762a2e3c3b: Writing region info on filesystem at 1733761853247Initializing all the Stores at 1733761853248 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761853248Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761853249 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761853249Cleaning up temporary data from old regions at 1733761853262 (+13 ms)Region opened successfully at 1733761853268 (+6 ms) 2024-12-09T16:30:53,268 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f906899ac9eecdfad6361a762a2e3c3b, disabling compactions & flushes 2024-12-09T16:30:53,268 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733761852714.f906899ac9eecdfad6361a762a2e3c3b. 2024-12-09T16:30:53,268 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733761852714.f906899ac9eecdfad6361a762a2e3c3b. 2024-12-09T16:30:53,268 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733761852714.f906899ac9eecdfad6361a762a2e3c3b. after waiting 0 ms 2024-12-09T16:30:53,268 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733761852714.f906899ac9eecdfad6361a762a2e3c3b. 2024-12-09T16:30:53,269 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733761852714.f906899ac9eecdfad6361a762a2e3c3b. 2024-12-09T16:30:53,269 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f906899ac9eecdfad6361a762a2e3c3b: Waiting for close lock at 1733761853268Disabling compacts and flushes for region at 1733761853268Disabling writes for close at 1733761853268Writing region close event to WAL at 1733761853269 (+1 ms)Closed at 1733761853269 2024-12-09T16:30:53,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741878_1055 (size=93) 2024-12-09T16:30:53,273 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-00071097/hregion-00071097.1733761852768 not finished, retry = 0 2024-12-09T16:30:53,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741878_1055 (size=93) 2024-12-09T16:30:53,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741878_1055 (size=93) 2024-12-09T16:30:53,377 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T16:30:53,377 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-00071097:(num 1733761852768) 2024-12-09T16:30:53,377 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T16:30:53,380 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733761852710, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:30:53,399 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733761852710/wal.1733761853381, exclude list is [], retry=0 2024-12-09T16:30:53,403 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:30:53,403 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:30:53,403 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:30:53,405 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733761852710/wal.1733761853381 2024-12-09T16:30:53,405 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39661:39661),(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:37099:37099)] 2024-12-09T16:30:53,406 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f906899ac9eecdfad6361a762a2e3c3b, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733761852714.f906899ac9eecdfad6361a762a2e3c3b.', STARTKEY => '', ENDKEY => ''} 2024-12-09T16:30:53,406 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733761852714.f906899ac9eecdfad6361a762a2e3c3b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:30:53,406 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:53,406 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:53,408 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:53,409 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f906899ac9eecdfad6361a762a2e3c3b columnFamilyName a 2024-12-09T16:30:53,409 DEBUG [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:53,410 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] regionserver.HStore(327): Store=f906899ac9eecdfad6361a762a2e3c3b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:53,410 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:53,411 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f906899ac9eecdfad6361a762a2e3c3b columnFamilyName b 2024-12-09T16:30:53,411 DEBUG [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:53,411 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] regionserver.HStore(327): Store=f906899ac9eecdfad6361a762a2e3c3b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:53,412 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:53,413 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f906899ac9eecdfad6361a762a2e3c3b columnFamilyName c 2024-12-09T16:30:53,413 DEBUG [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:53,413 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] regionserver.HStore(327): Store=f906899ac9eecdfad6361a762a2e3c3b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:53,414 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:53,414 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:53,416 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:53,417 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:53,417 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:53,418 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T16:30:53,419 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:53,420 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f906899ac9eecdfad6361a762a2e3c3b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59325892, jitterRate=-0.11597532033920288}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T16:30:53,421 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f906899ac9eecdfad6361a762a2e3c3b: Writing region info on filesystem at 1733761853406Initializing all the Stores at 1733761853407 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761853407Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761853408 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761853408Cleaning up temporary data from old regions at 1733761853417 (+9 ms)Region opened successfully at 1733761853421 (+4 ms) 2024-12-09T16:30:53,426 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733761853425/Put/seqid=0 2024-12-09T16:30:53,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741881_1058 (size=4826) 2024-12-09T16:30:53,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741881_1058 (size=4826) 2024-12-09T16:30:53,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741881_1058 (size=4826) 2024-12-09T16:30:53,433 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:39607/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in f906899ac9eecdfad6361a762a2e3c3b/a 2024-12-09T16:30:53,441 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-09T16:30:53,441 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T16:30:53,441 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f906899ac9eecdfad6361a762a2e3c3b: 2024-12-09T16:30:53,443 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b/a/595a8860435943f09a97765d3aad9d08_SeqId_3_ 2024-12-09T16:30:53,444 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:39607/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into f906899ac9eecdfad6361a762a2e3c3b/a as hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b/a/595a8860435943f09a97765d3aad9d08_SeqId_3_ - updating store file list. 2024-12-09T16:30:53,450 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 595a8860435943f09a97765d3aad9d08_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-09T16:30:53,451 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b/a/595a8860435943f09a97765d3aad9d08_SeqId_3_ into f906899ac9eecdfad6361a762a2e3c3b/a 2024-12-09T16:30:53,451 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:39607/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into f906899ac9eecdfad6361a762a2e3c3b/a (new location: hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b/a/595a8860435943f09a97765d3aad9d08_SeqId_3_) 2024-12-09T16:30:53,494 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39607/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733761852710/wal.1733761853381, size=0 (0bytes) 2024-12-09T16:30:53,494 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:39607/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733761852710/wal.1733761853381 might be still open, length is 0 2024-12-09T16:30:53,494 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39607/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733761852710/wal.1733761853381 2024-12-09T16:30:53,495 WARN [IPC Server handler 4 on default port 39607 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733761852710/wal.1733761853381 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-09T16:30:53,495 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39607/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733761852710/wal.1733761853381 after 1ms 2024-12-09T16:30:53,921 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:53782 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:39721:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53782 dst: /127.0.0.1:39721 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39721 remote=/127.0.0.1:53782]. Total timeout mills is 60000, 59537 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:30:53,922 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:51592 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:41451:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51592 dst: /127.0.0.1:41451 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:30:53,922 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:37602 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:36895:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37602 dst: /127.0.0.1:36895 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:30:53,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741880_1059 (size=434) 2024-12-09T16:30:53,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741880_1059 (size=434) 2024-12-09T16:30:57,496 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:39607/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733761852710/wal.1733761853381 after 4002ms 2024-12-09T16:30:57,499 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733761852710/wal.1733761853381: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:30:57,500 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39607/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733761852710/wal.1733761853381 took 4006ms 2024-12-09T16:30:57,502 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:39607/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733761852710/wal.1733761853381; continuing. 2024-12-09T16:30:57,502 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39607/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733761852710/wal.1733761853381 so closing down 2024-12-09T16:30:57,502 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T16:30:57,507 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733761853381.temp 2024-12-09T16:30:57,510 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b/recovered.edits/0000000000000000005-wal.1733761853381.temp 2024-12-09T16:30:57,510 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T16:30:57,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741882_1060 (size=236) 2024-12-09T16:30:57,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741882_1060 (size=236) 2024-12-09T16:30:57,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741882_1060 (size=236) 2024-12-09T16:30:57,524 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b/recovered.edits/0000000000000000005-wal.1733761853381.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-09T16:30:57,526 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b/recovered.edits/0000000000000000005-wal.1733761853381.temp to hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b/recovered.edits/0000000000000000005 2024-12-09T16:30:57,526 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 26 ms; skipped=1; WAL=hdfs://localhost:39607/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733761852710/wal.1733761853381, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T16:30:57,526 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39607/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733761852710/wal.1733761853381, journal: Splitting hdfs://localhost:39607/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733761852710/wal.1733761853381, size=0 (0bytes) at 1733761853494Finishing writing output for hdfs://localhost:39607/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733761852710/wal.1733761853381 so closing down at 1733761857502 (+4008 ms)Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b/recovered.edits/0000000000000000005-wal.1733761853381.temp at 1733761857510 (+8 ms)3 split writer threads finished at 1733761857510Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b/recovered.edits/0000000000000000005-wal.1733761853381.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733761857524 (+14 ms)Rename recovered edits hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b/recovered.edits/0000000000000000005-wal.1733761853381.temp to hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b/recovered.edits/0000000000000000005 at 1733761857526 (+2 ms)Processed 2 edits across 1 Regions in 26 ms; skipped=1; WAL=hdfs://localhost:39607/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733761852710/wal.1733761853381, size=0, length=0, corrupted=false, cancelled=false at 1733761857526 2024-12-09T16:30:57,528 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39607/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733761852710/wal.1733761853381 to hdfs://localhost:39607/hbase/oldWALs/wal.1733761853381 2024-12-09T16:30:57,529 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b/recovered.edits/0000000000000000005 2024-12-09T16:30:57,529 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T16:30:57,531 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733761852710, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:30:57,552 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733761852710/wal.1733761857531, exclude list is [], retry=0 2024-12-09T16:30:57,556 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:30:57,556 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:30:57,557 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:30:57,560 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733761852710/wal.1733761857531 2024-12-09T16:30:57,560 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:39661:39661),(127.0.0.1/127.0.0.1:37099:37099)] 2024-12-09T16:30:57,561 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f906899ac9eecdfad6361a762a2e3c3b, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733761852714.f906899ac9eecdfad6361a762a2e3c3b.', STARTKEY => '', ENDKEY => ''} 2024-12-09T16:30:57,561 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733761852714.f906899ac9eecdfad6361a762a2e3c3b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:30:57,561 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:57,561 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:57,563 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:57,564 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f906899ac9eecdfad6361a762a2e3c3b columnFamilyName a 2024-12-09T16:30:57,564 DEBUG [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:57,572 DEBUG [StoreFileOpener-f906899ac9eecdfad6361a762a2e3c3b-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 595a8860435943f09a97765d3aad9d08_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-09T16:30:57,572 DEBUG [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b/a/595a8860435943f09a97765d3aad9d08_SeqId_3_ 2024-12-09T16:30:57,572 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] regionserver.HStore(327): Store=f906899ac9eecdfad6361a762a2e3c3b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:57,572 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:57,574 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f906899ac9eecdfad6361a762a2e3c3b columnFamilyName b 2024-12-09T16:30:57,574 DEBUG [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:57,575 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] regionserver.HStore(327): Store=f906899ac9eecdfad6361a762a2e3c3b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:57,575 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:57,576 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f906899ac9eecdfad6361a762a2e3c3b columnFamilyName c 2024-12-09T16:30:57,576 DEBUG [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:57,577 INFO [StoreOpener-f906899ac9eecdfad6361a762a2e3c3b-1 {}] regionserver.HStore(327): Store=f906899ac9eecdfad6361a762a2e3c3b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:57,577 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:57,578 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:57,580 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:57,581 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b/recovered.edits/0000000000000000005 2024-12-09T16:30:57,584 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:30:57,585 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b/recovered.edits/0000000000000000005 2024-12-09T16:30:57,585 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f906899ac9eecdfad6361a762a2e3c3b 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-09T16:30:57,605 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b/.tmp/a/a6ae2240ad1245f7ba450a9ca0e03aa9 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733761853457/Put/seqid=0 2024-12-09T16:30:57,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741884_1062 (size=5149) 2024-12-09T16:30:57,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741884_1062 (size=5149) 2024-12-09T16:30:57,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741884_1062 (size=5149) 2024-12-09T16:30:57,616 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b/.tmp/a/a6ae2240ad1245f7ba450a9ca0e03aa9 2024-12-09T16:30:57,624 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b/.tmp/a/a6ae2240ad1245f7ba450a9ca0e03aa9 as hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b/a/a6ae2240ad1245f7ba450a9ca0e03aa9 2024-12-09T16:30:57,631 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b/a/a6ae2240ad1245f7ba450a9ca0e03aa9, entries=1, sequenceid=5, filesize=5.0 K 2024-12-09T16:30:57,631 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for f906899ac9eecdfad6361a762a2e3c3b in 46ms, sequenceid=5, compaction requested=false; wal=null 2024-12-09T16:30:57,632 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b/recovered.edits/0000000000000000005 2024-12-09T16:30:57,633 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:57,633 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:57,634 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T16:30:57,637 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f906899ac9eecdfad6361a762a2e3c3b 2024-12-09T16:30:57,639 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f906899ac9eecdfad6361a762a2e3c3b/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-09T16:30:57,640 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f906899ac9eecdfad6361a762a2e3c3b; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64421042, jitterRate=-0.040051668882369995}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T16:30:57,641 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f906899ac9eecdfad6361a762a2e3c3b: Writing region info on filesystem at 1733761857561Initializing all the Stores at 1733761857563 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761857563Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761857563Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761857563Obtaining lock to block concurrent updates at 1733761857585 (+22 ms)Preparing flush snapshotting stores in f906899ac9eecdfad6361a762a2e3c3b at 1733761857585Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733761852714.f906899ac9eecdfad6361a762a2e3c3b., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733761857585Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733761852714.f906899ac9eecdfad6361a762a2e3c3b. at 1733761857585Flushing f906899ac9eecdfad6361a762a2e3c3b/a: creating writer at 1733761857586 (+1 ms)Flushing f906899ac9eecdfad6361a762a2e3c3b/a: appending metadata at 1733761857605 (+19 ms)Flushing f906899ac9eecdfad6361a762a2e3c3b/a: closing flushed file at 1733761857605Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c2a9665: reopening flushed file at 1733761857623 (+18 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for f906899ac9eecdfad6361a762a2e3c3b in 46ms, sequenceid=5, compaction requested=false; wal=null at 1733761857631 (+8 ms)Cleaning up temporary data from old regions at 1733761857634 (+3 ms)Region opened successfully at 1733761857641 (+7 ms) 2024-12-09T16:30:57,645 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f906899ac9eecdfad6361a762a2e3c3b, disabling compactions & flushes 2024-12-09T16:30:57,645 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733761852714.f906899ac9eecdfad6361a762a2e3c3b. 2024-12-09T16:30:57,645 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733761852714.f906899ac9eecdfad6361a762a2e3c3b. 2024-12-09T16:30:57,645 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733761852714.f906899ac9eecdfad6361a762a2e3c3b. after waiting 0 ms 2024-12-09T16:30:57,645 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733761852714.f906899ac9eecdfad6361a762a2e3c3b. 2024-12-09T16:30:57,646 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733761852714.f906899ac9eecdfad6361a762a2e3c3b. 2024-12-09T16:30:57,646 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f906899ac9eecdfad6361a762a2e3c3b: Waiting for close lock at 1733761857645Disabling compacts and flushes for region at 1733761857645Disabling writes for close at 1733761857645Writing region close event to WAL at 1733761857646 (+1 ms)Closed at 1733761857646 2024-12-09T16:30:57,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741883_1061 (size=93) 2024-12-09T16:30:57,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741883_1061 (size=93) 2024-12-09T16:30:57,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741883_1061 (size=93) 2024-12-09T16:30:57,654 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T16:30:57,654 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733761857531) 2024-12-09T16:30:57,665 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=418 (was 408) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:41135 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-143335312_22 at /127.0.0.1:33742 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/cluster_cd98e670-d246-b6f3-ff89-5ca1891baf9f/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/cluster_cd98e670-d246-b6f3-ff89-5ca1891baf9f/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-143335312_22 at /127.0.0.1:56052 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-143335312_22 at /127.0.0.1:56620 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41135 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:39607 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:39607 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1025 (was 959) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=636 (was 665), ProcessCount=11 (was 11), AvailableMemoryMB=6435 (was 6439) 2024-12-09T16:30:57,667 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-12-09T16:30:57,678 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=418, OpenFileDescriptor=1025, MaxFileDescriptor=1048576, SystemLoadAverage=636, ProcessCount=11, AvailableMemoryMB=6434 2024-12-09T16:30:57,678 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-12-09T16:30:57,695 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:30:57,701 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T16:30:57,705 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is dfce6ce5b10c,35827,1733761839880 2024-12-09T16:30:57,707 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@8d821fe 2024-12-09T16:30:57,709 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T16:30:57,711 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T16:30:57,714 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35827 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T16:30:57,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35827 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-09T16:30:57,722 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T16:30:57,724 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35827 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-09T16:30:57,724 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:57,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35827 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T16:30:57,734 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T16:30:57,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741885_1063 (size=694) 2024-12-09T16:30:57,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741885_1063 (size=694) 2024-12-09T16:30:57,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741885_1063 (size=694) 2024-12-09T16:30:57,749 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f010e6e532012a33c4c8a2a892bd1818, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38 2024-12-09T16:30:57,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741886_1064 (size=77) 2024-12-09T16:30:57,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741886_1064 (size=77) 2024-12-09T16:30:57,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741886_1064 (size=77) 2024-12-09T16:30:57,758 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:30:57,758 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing f010e6e532012a33c4c8a2a892bd1818, disabling compactions & flushes 2024-12-09T16:30:57,758 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:57,758 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:57,758 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. after waiting 0 ms 2024-12-09T16:30:57,758 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:57,758 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:57,758 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for f010e6e532012a33c4c8a2a892bd1818: Waiting for close lock at 1733761857758Disabling compacts and flushes for region at 1733761857758Disabling writes for close at 1733761857758Writing region close event to WAL at 1733761857758Closed at 1733761857758 2024-12-09T16:30:57,760 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T16:30:57,766 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733761857760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733761857760"}]},"ts":"1733761857760"} 2024-12-09T16:30:57,769 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T16:30:57,771 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T16:30:57,774 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733761857771"}]},"ts":"1733761857771"} 2024-12-09T16:30:57,778 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-09T16:30:57,779 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {dfce6ce5b10c=0} racks are {/default-rack=0} 2024-12-09T16:30:57,780 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T16:30:57,780 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T16:30:57,780 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-09T16:30:57,780 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T16:30:57,780 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T16:30:57,780 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-09T16:30:57,780 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T16:30:57,780 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T16:30:57,780 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-09T16:30:57,780 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T16:30:57,782 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f010e6e532012a33c4c8a2a892bd1818, ASSIGN}] 2024-12-09T16:30:57,784 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f010e6e532012a33c4c8a2a892bd1818, ASSIGN 2024-12-09T16:30:57,785 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f010e6e532012a33c4c8a2a892bd1818, ASSIGN; state=OFFLINE, location=dfce6ce5b10c,32855,1733761840798; forceNewPlan=false, retain=false 2024-12-09T16:30:57,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35827 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T16:30:57,937 INFO [dfce6ce5b10c:35827 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T16:30:57,938 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f010e6e532012a33c4c8a2a892bd1818, regionState=OPENING, regionLocation=dfce6ce5b10c,32855,1733761840798 2024-12-09T16:30:57,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f010e6e532012a33c4c8a2a892bd1818, ASSIGN because future has completed 2024-12-09T16:30:57,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f010e6e532012a33c4c8a2a892bd1818, server=dfce6ce5b10c,32855,1733761840798}] 2024-12-09T16:30:58,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35827 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T16:30:58,097 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T16:30:58,099 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46609, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T16:30:58,106 INFO [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:58,106 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f010e6e532012a33c4c8a2a892bd1818, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818.', STARTKEY => '', ENDKEY => ''} 2024-12-09T16:30:58,107 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,107 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:30:58,107 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,107 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,110 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,112 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f010e6e532012a33c4c8a2a892bd1818 columnFamilyName cf1 2024-12-09T16:30:58,112 DEBUG [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:58,113 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.HStore(327): Store=f010e6e532012a33c4c8a2a892bd1818/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:58,113 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,115 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f010e6e532012a33c4c8a2a892bd1818 columnFamilyName cf2 2024-12-09T16:30:58,115 DEBUG [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:58,116 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.HStore(327): Store=f010e6e532012a33c4c8a2a892bd1818/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:58,116 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,117 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,117 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,118 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,118 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,119 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T16:30:58,121 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,124 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T16:30:58,125 INFO [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f010e6e532012a33c4c8a2a892bd1818; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72437645, jitterRate=0.07940502464771271}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T16:30:58,125 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,126 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f010e6e532012a33c4c8a2a892bd1818: Running coprocessor pre-open hook at 1733761858108Writing region info on filesystem at 1733761858108Initializing all the Stores at 1733761858109 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761858109Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761858109Cleaning up temporary data from old regions at 1733761858118 (+9 ms)Running coprocessor post-open hooks at 1733761858125 (+7 ms)Region opened successfully at 1733761858126 (+1 ms) 2024-12-09T16:30:58,128 INFO [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., pid=6, masterSystemTime=1733761858097 2024-12-09T16:30:58,131 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:58,131 INFO [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:58,132 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f010e6e532012a33c4c8a2a892bd1818, regionState=OPEN, openSeqNum=2, regionLocation=dfce6ce5b10c,32855,1733761840798 2024-12-09T16:30:58,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f010e6e532012a33c4c8a2a892bd1818, server=dfce6ce5b10c,32855,1733761840798 because future has completed 2024-12-09T16:30:58,144 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-09T16:30:58,144 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f010e6e532012a33c4c8a2a892bd1818, server=dfce6ce5b10c,32855,1733761840798 in 197 msec 2024-12-09T16:30:58,148 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-09T16:30:58,148 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f010e6e532012a33c4c8a2a892bd1818, ASSIGN in 362 msec 2024-12-09T16:30:58,150 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T16:30:58,150 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733761858150"}]},"ts":"1733761858150"} 2024-12-09T16:30:58,153 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-09T16:30:58,155 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T16:30:58,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 439 msec 2024-12-09T16:30:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35827 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T16:30:58,360 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-09T16:30:58,360 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-09T16:30:58,362 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T16:30:58,368 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-09T16:30:58,369 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T16:30:58,370 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-09T16:30:58,385 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,32855,1733761840798, seqNum=2] 2024-12-09T16:30:58,387 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T16:30:58,389 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T16:30:58,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35827 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=f010e6e532012a33c4c8a2a892bd1818, source=dfce6ce5b10c,32855,1733761840798, destination=dfce6ce5b10c,38861,1733761840646, warming up region on dfce6ce5b10c,38861,1733761840646 2024-12-09T16:30:58,409 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35827 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=f010e6e532012a33c4c8a2a892bd1818, source=dfce6ce5b10c,32855,1733761840798, destination=dfce6ce5b10c,38861,1733761840646, running balancer 2024-12-09T16:30:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35827 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f010e6e532012a33c4c8a2a892bd1818, REOPEN/MOVE 2024-12-09T16:30:58,411 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f010e6e532012a33c4c8a2a892bd1818, REOPEN/MOVE 2024-12-09T16:30:58,413 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38861 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38861 {}] regionserver.HRegion(7855): Warmup {ENCODED => f010e6e532012a33c4c8a2a892bd1818, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818.', STARTKEY => '', ENDKEY => ''} 2024-12-09T16:30:58,414 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f010e6e532012a33c4c8a2a892bd1818, regionState=CLOSING, regionLocation=dfce6ce5b10c,32855,1733761840798 2024-12-09T16:30:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38861 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:30:58,415 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,416 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f010e6e532012a33c4c8a2a892bd1818 columnFamilyName cf1 2024-12-09T16:30:58,416 DEBUG [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:58,417 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.HStore(327): Store=f010e6e532012a33c4c8a2a892bd1818/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:58,417 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,418 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f010e6e532012a33c4c8a2a892bd1818, REOPEN/MOVE because future has completed 2024-12-09T16:30:58,418 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f010e6e532012a33c4c8a2a892bd1818 columnFamilyName cf2 2024-12-09T16:30:58,418 DEBUG [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:58,418 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T16:30:58,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure f010e6e532012a33c4c8a2a892bd1818, server=dfce6ce5b10c,32855,1733761840798}] 2024-12-09T16:30:58,419 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.HStore(327): Store=f010e6e532012a33c4c8a2a892bd1818/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38861 {}] regionserver.HRegion(1722): Closing f010e6e532012a33c4c8a2a892bd1818, disabling compactions & flushes 2024-12-09T16:30:58,419 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38861 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38861 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38861 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. after waiting 0 ms 2024-12-09T16:30:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38861 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:58,422 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38861 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38861 {}] regionserver.HRegion(1676): Region close journal for f010e6e532012a33c4c8a2a892bd1818: Waiting for close lock at 1733761858419Disabling compacts and flushes for region at 1733761858419Disabling writes for close at 1733761858419Writing region close event to WAL at 1733761858422 (+3 ms)Closed at 1733761858422 2024-12-09T16:30:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35827 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-09T16:30:58,578 INFO [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,578 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T16:30:58,578 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing f010e6e532012a33c4c8a2a892bd1818, disabling compactions & flushes 2024-12-09T16:30:58,578 INFO [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:58,579 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:58,579 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. after waiting 0 ms 2024-12-09T16:30:58,579 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:58,579 INFO [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing f010e6e532012a33c4c8a2a892bd1818 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-09T16:30:58,594 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/.tmp/cf1/1a6ee832880d4c3da652c06857257d66 is 35, key is r1/cf1:q/1733761858390/Put/seqid=0 2024-12-09T16:30:58,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741887_1065 (size=4783) 2024-12-09T16:30:58,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741887_1065 (size=4783) 2024-12-09T16:30:58,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741887_1065 (size=4783) 2024-12-09T16:30:58,605 INFO [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/.tmp/cf1/1a6ee832880d4c3da652c06857257d66 2024-12-09T16:30:58,613 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/.tmp/cf1/1a6ee832880d4c3da652c06857257d66 as hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf1/1a6ee832880d4c3da652c06857257d66 2024-12-09T16:30:58,621 INFO [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf1/1a6ee832880d4c3da652c06857257d66, entries=1, sequenceid=5, filesize=4.7 K 2024-12-09T16:30:58,622 INFO [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for f010e6e532012a33c4c8a2a892bd1818 in 43ms, sequenceid=5, compaction requested=false 2024-12-09T16:30:58,622 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-09T16:30:58,628 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-09T16:30:58,631 INFO [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:58,632 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for f010e6e532012a33c4c8a2a892bd1818: Waiting for close lock at 1733761858578Running coprocessor pre-close hooks at 1733761858578Disabling compacts and flushes for region at 1733761858578Disabling writes for close at 1733761858579 (+1 ms)Obtaining lock to block concurrent updates at 1733761858579Preparing flush snapshotting stores in f010e6e532012a33c4c8a2a892bd1818 at 1733761858579Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733761858579Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. at 1733761858580 (+1 ms)Flushing f010e6e532012a33c4c8a2a892bd1818/cf1: creating writer at 1733761858580Flushing f010e6e532012a33c4c8a2a892bd1818/cf1: appending metadata at 1733761858594 (+14 ms)Flushing f010e6e532012a33c4c8a2a892bd1818/cf1: closing flushed file at 1733761858594Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@736742c7: reopening flushed file at 1733761858612 (+18 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for f010e6e532012a33c4c8a2a892bd1818 in 43ms, sequenceid=5, compaction requested=false at 1733761858622 (+10 ms)Writing region close event to WAL at 1733761858623 (+1 ms)Running coprocessor post-close hooks at 1733761858629 (+6 ms)Closed at 1733761858631 (+2 ms) 2024-12-09T16:30:58,632 INFO [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding f010e6e532012a33c4c8a2a892bd1818 move to dfce6ce5b10c,38861,1733761840646 record at close sequenceid=5 2024-12-09T16:30:58,636 INFO [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,638 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f010e6e532012a33c4c8a2a892bd1818, regionState=CLOSED 2024-12-09T16:30:58,641 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure f010e6e532012a33c4c8a2a892bd1818, server=dfce6ce5b10c,32855,1733761840798 because future has completed 2024-12-09T16:30:58,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-09T16:30:58,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure f010e6e532012a33c4c8a2a892bd1818, server=dfce6ce5b10c,32855,1733761840798 in 224 msec 2024-12-09T16:30:58,648 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f010e6e532012a33c4c8a2a892bd1818, REOPEN/MOVE; state=CLOSED, location=dfce6ce5b10c,38861,1733761840646; forceNewPlan=false, retain=false 2024-12-09T16:30:58,799 INFO [dfce6ce5b10c:35827 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T16:30:58,799 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f010e6e532012a33c4c8a2a892bd1818, regionState=OPENING, regionLocation=dfce6ce5b10c,38861,1733761840646 2024-12-09T16:30:58,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f010e6e532012a33c4c8a2a892bd1818, REOPEN/MOVE because future has completed 2024-12-09T16:30:58,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure f010e6e532012a33c4c8a2a892bd1818, server=dfce6ce5b10c,38861,1733761840646}] 2024-12-09T16:30:58,964 INFO [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:58,964 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => f010e6e532012a33c4c8a2a892bd1818, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818.', STARTKEY => '', ENDKEY => ''} 2024-12-09T16:30:58,964 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,965 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:30:58,965 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,965 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,967 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,970 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f010e6e532012a33c4c8a2a892bd1818 columnFamilyName cf1 2024-12-09T16:30:58,970 DEBUG [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:58,988 DEBUG [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf1/1a6ee832880d4c3da652c06857257d66 2024-12-09T16:30:58,988 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.HStore(327): Store=f010e6e532012a33c4c8a2a892bd1818/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:58,988 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,990 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f010e6e532012a33c4c8a2a892bd1818 columnFamilyName cf2 2024-12-09T16:30:58,990 DEBUG [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:58,991 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.HStore(327): Store=f010e6e532012a33c4c8a2a892bd1818/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:58,991 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,992 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,995 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,996 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,996 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:58,997 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T16:30:59,000 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:59,001 INFO [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened f010e6e532012a33c4c8a2a892bd1818; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71164780, jitterRate=0.06043785810470581}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T16:30:59,001 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:59,003 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for f010e6e532012a33c4c8a2a892bd1818: Running coprocessor pre-open hook at 1733761858965Writing region info on filesystem at 1733761858965Initializing all the Stores at 1733761858966 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761858966Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761858967 (+1 ms)Cleaning up temporary data from old regions at 1733761858996 (+29 ms)Running coprocessor post-open hooks at 1733761859002 (+6 ms)Region opened successfully at 1733761859002 2024-12-09T16:30:59,006 INFO [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., pid=9, masterSystemTime=1733761858958 2024-12-09T16:30:59,010 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:59,010 INFO [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:59,012 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f010e6e532012a33c4c8a2a892bd1818, regionState=OPEN, openSeqNum=9, regionLocation=dfce6ce5b10c,38861,1733761840646 2024-12-09T16:30:59,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure f010e6e532012a33c4c8a2a892bd1818, server=dfce6ce5b10c,38861,1733761840646 because future has completed 2024-12-09T16:30:59,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-09T16:30:59,025 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure f010e6e532012a33c4c8a2a892bd1818, server=dfce6ce5b10c,38861,1733761840646 in 214 msec 2024-12-09T16:30:59,030 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f010e6e532012a33c4c8a2a892bd1818, REOPEN/MOVE in 615 msec 2024-12-09T16:30:59,070 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T16:30:59,072 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T16:30:59,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32855 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:33068 deadline: 1733761919079, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=dfce6ce5b10c port=38861 startCode=1733761840646. As of locationSeqNum=5. 2024-12-09T16:30:59,115 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,32855,1733761840798, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,32855,1733761840798, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=dfce6ce5b10c port=38861 startCode=1733761840646. As of locationSeqNum=5. 2024-12-09T16:30:59,116 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,32855,1733761840798, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=dfce6ce5b10c port=38861 startCode=1733761840646. As of locationSeqNum=5. 2024-12-09T16:30:59,117 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,32855,1733761840798, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,38861,1733761840646, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=dfce6ce5b10c port=38861 startCode=1733761840646. As of locationSeqNum=5. 2024-12-09T16:30:59,239 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f010e6e532012a33c4c8a2a892bd1818 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-09T16:30:59,256 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/.tmp/cf1/c4a89785b2284955940eb7b69b95afa9 is 29, key is r1/cf1:/1733761859231/DeleteFamily/seqid=0 2024-12-09T16:30:59,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741888_1066 (size=4906) 2024-12-09T16:30:59,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741888_1066 (size=4906) 2024-12-09T16:30:59,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741888_1066 (size=4906) 2024-12-09T16:30:59,270 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/.tmp/cf1/c4a89785b2284955940eb7b69b95afa9 2024-12-09T16:30:59,278 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c4a89785b2284955940eb7b69b95afa9 2024-12-09T16:30:59,293 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/.tmp/cf2/fff4b43f664c4a3eb0df64bbd7288a23 is 29, key is r1/cf2:/1733761859231/DeleteFamily/seqid=0 2024-12-09T16:30:59,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741889_1067 (size=4906) 2024-12-09T16:30:59,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741889_1067 (size=4906) 2024-12-09T16:30:59,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741889_1067 (size=4906) 2024-12-09T16:30:59,305 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/.tmp/cf2/fff4b43f664c4a3eb0df64bbd7288a23 2024-12-09T16:30:59,312 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fff4b43f664c4a3eb0df64bbd7288a23 2024-12-09T16:30:59,313 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/.tmp/cf1/c4a89785b2284955940eb7b69b95afa9 as hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf1/c4a89785b2284955940eb7b69b95afa9 2024-12-09T16:30:59,320 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c4a89785b2284955940eb7b69b95afa9 2024-12-09T16:30:59,321 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf1/c4a89785b2284955940eb7b69b95afa9, entries=1, sequenceid=12, filesize=4.8 K 2024-12-09T16:30:59,322 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/.tmp/cf2/fff4b43f664c4a3eb0df64bbd7288a23 as hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf2/fff4b43f664c4a3eb0df64bbd7288a23 2024-12-09T16:30:59,329 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fff4b43f664c4a3eb0df64bbd7288a23 2024-12-09T16:30:59,329 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf2/fff4b43f664c4a3eb0df64bbd7288a23, entries=1, sequenceid=12, filesize=4.8 K 2024-12-09T16:30:59,330 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for f010e6e532012a33c4c8a2a892bd1818 in 91ms, sequenceid=12, compaction requested=false 2024-12-09T16:30:59,330 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f010e6e532012a33c4c8a2a892bd1818: 2024-12-09T16:30:59,333 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T16:30:59,334 DEBUG [Time-limited test {}] regionserver.HStore(1541): f010e6e532012a33c4c8a2a892bd1818/cf1 is initiating major compaction (all files) 2024-12-09T16:30:59,334 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T16:30:59,334 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:59,335 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of f010e6e532012a33c4c8a2a892bd1818/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:59,335 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf1/1a6ee832880d4c3da652c06857257d66, hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf1/c4a89785b2284955940eb7b69b95afa9] into tmpdir=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/.tmp, totalSize=9.5 K 2024-12-09T16:30:59,336 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1a6ee832880d4c3da652c06857257d66, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733761858390 2024-12-09T16:30:59,337 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c4a89785b2284955940eb7b69b95afa9, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-09T16:30:59,348 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): f010e6e532012a33c4c8a2a892bd1818#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T16:30:59,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741890_1068 (size=4626) 2024-12-09T16:30:59,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741890_1068 (size=4626) 2024-12-09T16:30:59,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741890_1068 (size=4626) 2024-12-09T16:30:59,368 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/.tmp/cf1/e72135fa8984470d855ea27665b50e2b as hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf1/e72135fa8984470d855ea27665b50e2b 2024-12-09T16:30:59,382 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in f010e6e532012a33c4c8a2a892bd1818/cf1 of f010e6e532012a33c4c8a2a892bd1818 into e72135fa8984470d855ea27665b50e2b(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T16:30:59,382 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for f010e6e532012a33c4c8a2a892bd1818: 2024-12-09T16:30:59,382 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-09T16:30:59,383 DEBUG [Time-limited test {}] regionserver.HStore(1541): f010e6e532012a33c4c8a2a892bd1818/cf2 is initiating major compaction (all files) 2024-12-09T16:30:59,383 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T16:30:59,383 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T16:30:59,383 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of f010e6e532012a33c4c8a2a892bd1818/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:59,383 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf2/fff4b43f664c4a3eb0df64bbd7288a23] into tmpdir=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/.tmp, totalSize=4.8 K 2024-12-09T16:30:59,384 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting fff4b43f664c4a3eb0df64bbd7288a23, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-09T16:30:59,390 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): f010e6e532012a33c4c8a2a892bd1818#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T16:30:59,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741891_1069 (size=4592) 2024-12-09T16:30:59,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741891_1069 (size=4592) 2024-12-09T16:30:59,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741891_1069 (size=4592) 2024-12-09T16:30:59,409 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/.tmp/cf2/f5c013cde7ad4c8c86a9a6dc59d717d9 as hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf2/f5c013cde7ad4c8c86a9a6dc59d717d9 2024-12-09T16:30:59,422 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in f010e6e532012a33c4c8a2a892bd1818/cf2 of f010e6e532012a33c4c8a2a892bd1818 into f5c013cde7ad4c8c86a9a6dc59d717d9(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T16:30:59,422 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for f010e6e532012a33c4c8a2a892bd1818: 2024-12-09T16:30:59,428 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35827 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=f010e6e532012a33c4c8a2a892bd1818, source=dfce6ce5b10c,38861,1733761840646, destination=dfce6ce5b10c,32855,1733761840798, warming up region on dfce6ce5b10c,32855,1733761840798 2024-12-09T16:30:59,428 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35827 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=f010e6e532012a33c4c8a2a892bd1818, source=dfce6ce5b10c,38861,1733761840646, destination=dfce6ce5b10c,32855,1733761840798, running balancer 2024-12-09T16:30:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35827 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f010e6e532012a33c4c8a2a892bd1818, REOPEN/MOVE 2024-12-09T16:30:59,430 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f010e6e532012a33c4c8a2a892bd1818, REOPEN/MOVE 2024-12-09T16:30:59,432 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f010e6e532012a33c4c8a2a892bd1818, regionState=CLOSING, regionLocation=dfce6ce5b10c,38861,1733761840646 2024-12-09T16:30:59,433 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32855 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32855 {}] regionserver.HRegion(7855): Warmup {ENCODED => f010e6e532012a33c4c8a2a892bd1818, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818.', STARTKEY => '', ENDKEY => ''} 2024-12-09T16:30:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32855 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:30:59,434 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:59,435 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f010e6e532012a33c4c8a2a892bd1818 columnFamilyName cf1 2024-12-09T16:30:59,435 DEBUG [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:59,437 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f010e6e532012a33c4c8a2a892bd1818, REOPEN/MOVE because future has completed 2024-12-09T16:30:59,437 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T16:30:59,437 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure f010e6e532012a33c4c8a2a892bd1818, server=dfce6ce5b10c,38861,1733761840646}] 2024-12-09T16:30:59,446 DEBUG [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf1/1a6ee832880d4c3da652c06857257d66 2024-12-09T16:30:59,452 INFO [StoreFileOpener-f010e6e532012a33c4c8a2a892bd1818-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c4a89785b2284955940eb7b69b95afa9 2024-12-09T16:30:59,453 DEBUG [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf1/c4a89785b2284955940eb7b69b95afa9 2024-12-09T16:30:59,460 DEBUG [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf1/e72135fa8984470d855ea27665b50e2b 2024-12-09T16:30:59,460 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.HStore(327): Store=f010e6e532012a33c4c8a2a892bd1818/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:59,460 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:59,461 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f010e6e532012a33c4c8a2a892bd1818 columnFamilyName cf2 2024-12-09T16:30:59,462 DEBUG [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:59,469 DEBUG [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf2/f5c013cde7ad4c8c86a9a6dc59d717d9 2024-12-09T16:30:59,475 INFO [StoreFileOpener-f010e6e532012a33c4c8a2a892bd1818-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fff4b43f664c4a3eb0df64bbd7288a23 2024-12-09T16:30:59,475 DEBUG [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf2/fff4b43f664c4a3eb0df64bbd7288a23 2024-12-09T16:30:59,475 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.HStore(327): Store=f010e6e532012a33c4c8a2a892bd1818/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32855 {}] regionserver.HRegion(1722): Closing f010e6e532012a33c4c8a2a892bd1818, disabling compactions & flushes 2024-12-09T16:30:59,476 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32855 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32855 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32855 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. after waiting 0 ms 2024-12-09T16:30:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32855 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:59,477 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32855 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:59,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32855 {}] regionserver.HRegion(1676): Region close journal for f010e6e532012a33c4c8a2a892bd1818: Waiting for close lock at 1733761859476Disabling compacts and flushes for region at 1733761859476Disabling writes for close at 1733761859476Writing region close event to WAL at 1733761859477 (+1 ms)Closed at 1733761859477 2024-12-09T16:30:59,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35827 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-09T16:30:59,592 INFO [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:59,592 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T16:30:59,593 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing f010e6e532012a33c4c8a2a892bd1818, disabling compactions & flushes 2024-12-09T16:30:59,593 INFO [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:59,593 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:59,593 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. after waiting 0 ms 2024-12-09T16:30:59,593 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:59,593 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf1/1a6ee832880d4c3da652c06857257d66, hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf1/c4a89785b2284955940eb7b69b95afa9] to archive 2024-12-09T16:30:59,596 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T16:30:59,600 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf1/1a6ee832880d4c3da652c06857257d66 to hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf1/1a6ee832880d4c3da652c06857257d66 2024-12-09T16:30:59,602 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf1/c4a89785b2284955940eb7b69b95afa9 to hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf1/c4a89785b2284955940eb7b69b95afa9 2024-12-09T16:30:59,616 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf2/fff4b43f664c4a3eb0df64bbd7288a23] to archive 2024-12-09T16:30:59,617 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T16:30:59,619 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf2/fff4b43f664c4a3eb0df64bbd7288a23 to hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf2/fff4b43f664c4a3eb0df64bbd7288a23 2024-12-09T16:30:59,625 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-09T16:30:59,625 INFO [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:59,626 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for f010e6e532012a33c4c8a2a892bd1818: Waiting for close lock at 1733761859593Running coprocessor pre-close hooks at 1733761859593Disabling compacts and flushes for region at 1733761859593Disabling writes for close at 1733761859593Writing region close event to WAL at 1733761859621 (+28 ms)Running coprocessor post-close hooks at 1733761859625 (+4 ms)Closed at 1733761859625 2024-12-09T16:30:59,626 INFO [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding f010e6e532012a33c4c8a2a892bd1818 move to dfce6ce5b10c,32855,1733761840798 record at close sequenceid=12 2024-12-09T16:30:59,628 INFO [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:59,629 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f010e6e532012a33c4c8a2a892bd1818, regionState=CLOSED 2024-12-09T16:30:59,631 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure f010e6e532012a33c4c8a2a892bd1818, server=dfce6ce5b10c,38861,1733761840646 because future has completed 2024-12-09T16:30:59,636 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-09T16:30:59,636 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure f010e6e532012a33c4c8a2a892bd1818, server=dfce6ce5b10c,38861,1733761840646 in 196 msec 2024-12-09T16:30:59,637 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f010e6e532012a33c4c8a2a892bd1818, REOPEN/MOVE; state=CLOSED, location=dfce6ce5b10c,32855,1733761840798; forceNewPlan=false, retain=false 2024-12-09T16:30:59,788 INFO [dfce6ce5b10c:35827 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T16:30:59,788 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f010e6e532012a33c4c8a2a892bd1818, regionState=OPENING, regionLocation=dfce6ce5b10c,32855,1733761840798 2024-12-09T16:30:59,791 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f010e6e532012a33c4c8a2a892bd1818, REOPEN/MOVE because future has completed 2024-12-09T16:30:59,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure f010e6e532012a33c4c8a2a892bd1818, server=dfce6ce5b10c,32855,1733761840798}] 2024-12-09T16:30:59,948 INFO [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:59,948 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => f010e6e532012a33c4c8a2a892bd1818, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818.', STARTKEY => '', ENDKEY => ''} 2024-12-09T16:30:59,949 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:59,949 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:30:59,949 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:59,949 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:59,950 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:59,951 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f010e6e532012a33c4c8a2a892bd1818 columnFamilyName cf1 2024-12-09T16:30:59,952 DEBUG [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:59,957 DEBUG [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf1/e72135fa8984470d855ea27665b50e2b 2024-12-09T16:30:59,958 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.HStore(327): Store=f010e6e532012a33c4c8a2a892bd1818/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:59,958 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:59,959 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f010e6e532012a33c4c8a2a892bd1818 columnFamilyName cf2 2024-12-09T16:30:59,959 DEBUG [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:30:59,966 DEBUG [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf2/f5c013cde7ad4c8c86a9a6dc59d717d9 2024-12-09T16:30:59,967 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.HStore(327): Store=f010e6e532012a33c4c8a2a892bd1818/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:30:59,967 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:59,968 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:59,969 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:59,970 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:59,970 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:59,971 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T16:30:59,972 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:59,973 INFO [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened f010e6e532012a33c4c8a2a892bd1818; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62651316, jitterRate=-0.06642264127731323}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T16:30:59,974 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:30:59,975 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for f010e6e532012a33c4c8a2a892bd1818: Running coprocessor pre-open hook at 1733761859949Writing region info on filesystem at 1733761859949Initializing all the Stores at 1733761859950 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761859950Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761859950Cleaning up temporary data from old regions at 1733761859970 (+20 ms)Running coprocessor post-open hooks at 1733761859974 (+4 ms)Region opened successfully at 1733761859974 2024-12-09T16:30:59,976 INFO [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., pid=12, masterSystemTime=1733761859944 2024-12-09T16:30:59,979 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:59,979 INFO [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:30:59,980 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f010e6e532012a33c4c8a2a892bd1818, regionState=OPEN, openSeqNum=18, regionLocation=dfce6ce5b10c,32855,1733761840798 2024-12-09T16:30:59,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure f010e6e532012a33c4c8a2a892bd1818, server=dfce6ce5b10c,32855,1733761840798 because future has completed 2024-12-09T16:30:59,987 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-09T16:30:59,987 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure f010e6e532012a33c4c8a2a892bd1818, server=dfce6ce5b10c,32855,1733761840798 in 192 msec 2024-12-09T16:30:59,990 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f010e6e532012a33c4c8a2a892bd1818, REOPEN/MOVE in 559 msec 2024-12-09T16:31:00,032 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T16:31:00,034 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T16:31:00,036 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server dfce6ce5b10c,32855,1733761840798: testing ***** 2024-12-09T16:31:00,036 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-09T16:31:00,038 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-09T16:31:00,040 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-09T16:31:00,045 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-09T16:31:00,047 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-09T16:31:00,059 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 331915256 }, "NonHeapMemoryUsage": { "committed": 171638784, "init": 7667712, "max": -1, "used": 168867952 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "dfce6ce5b10c", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 4, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 10271, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 48, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 3, "ProcessCallTime_90th_percentile": 6, "ProcessCallTime_95th_percentile": 13, "ProcessCallTime_98th_percentile": 48, "ProcessCallTime_99th_percentile": 48, "ProcessCallTime_99.9th_percentile": 48, "ProcessCallTime_TimeRangeCount_0-1": 12, "ProcessCallTime_TimeRangeCount_1-3": 12, "ProcessCallTime_TimeRangeCount_3-10": 5, "ProcessCallTime_TimeRangeCount_10-30": 1, "ProcessCallTime_TimeRangeCount_30-100": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 48, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 3, "TotalCallTime_90th_percentile": 6, "TotalCallTime_95th_percentile": 13, "TotalCallTime_98th_percentile": 48, "TotalCallTime_99th_percentile": 48, "TotalCallTime_99.9th_percentile": 48, "TotalCallTime_TimeRangeCount_0-1": 12, "TotalCallTime_TimeRangeCount_1-3": 12, "TotalCallTime_TimeRangeCount_3-10": 5, "TotalCallTime_TimeRangeCount_10-30": 1, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1138, "ResponseSize_mean": 159, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 795, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1069, "ResponseSize_99.9th_percentile": 1131, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5772 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "dfce6ce5b10c", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:60962", "tag.serverName": "dfce6ce5b10c,34709,1733761840861", "tag.clusterId": "80e51bf9-2430-42cd-8179-eacd4b0999ae", "tag.Context": "regionserver", "tag.Hostname": "dfce6ce5b10c", "regionCount": 0, "storeCount": 0, "hlogFileCount": 3, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733761840861, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227243, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 1.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-09T16:31:00,063 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35827 {}] master.MasterRpcServices(700): dfce6ce5b10c,32855,1733761840798 reported a fatal error: ***** ABORTING region server dfce6ce5b10c,32855,1733761840798: testing ***** 2024-12-09T16:31:00,065 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dfce6ce5b10c,32855,1733761840798' ***** 2024-12-09T16:31:00,065 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-09T16:31:00,066 INFO [RS:1;dfce6ce5b10c:32855 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T16:31:00,066 INFO [RS:1;dfce6ce5b10c:32855 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-09T16:31:00,066 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T16:31:00,066 INFO [RS:1;dfce6ce5b10c:32855 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-09T16:31:00,066 INFO [RS:1;dfce6ce5b10c:32855 {}] regionserver.HRegionServer(3091): Received CLOSE for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:31:00,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38861 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:36510 deadline: 1733761920066, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=dfce6ce5b10c port=32855 startCode=1733761840798. As of locationSeqNum=12. 2024-12-09T16:31:00,067 INFO [RS:1;dfce6ce5b10c:32855 {}] regionserver.HRegionServer(956): aborting server dfce6ce5b10c,32855,1733761840798 2024-12-09T16:31:00,067 INFO [RS:1;dfce6ce5b10c:32855 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T16:31:00,067 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f010e6e532012a33c4c8a2a892bd1818, disabling compactions & flushes 2024-12-09T16:31:00,067 INFO [RS:1;dfce6ce5b10c:32855 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;dfce6ce5b10c:32855. 2024-12-09T16:31:00,067 INFO [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:31:00,067 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:31:00,067 DEBUG [RS:1;dfce6ce5b10c:32855 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T16:31:00,067 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. after waiting 0 ms 2024-12-09T16:31:00,067 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,38861,1733761840646, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,38861,1733761840646, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=dfce6ce5b10c port=32855 startCode=1733761840798. As of locationSeqNum=12. 2024-12-09T16:31:00,067 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:31:00,067 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,38861,1733761840646, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=dfce6ce5b10c port=32855 startCode=1733761840798. As of locationSeqNum=12. 2024-12-09T16:31:00,068 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,38861,1733761840646, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,32855,1733761840798, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=dfce6ce5b10c port=32855 startCode=1733761840798. As of locationSeqNum=12. 2024-12-09T16:31:00,068 DEBUG [RS:1;dfce6ce5b10c:32855 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T16:31:00,068 INFO [RS:1;dfce6ce5b10c:32855 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T16:31:00,068 DEBUG [RS:1;dfce6ce5b10c:32855 {}] regionserver.HRegionServer(1325): Online Regions={f010e6e532012a33c4c8a2a892bd1818=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818.} 2024-12-09T16:31:00,069 DEBUG [RS:1;dfce6ce5b10c:32855 {}] regionserver.HRegionServer(1351): Waiting on f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:31:00,070 INFO [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:31:00,070 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f010e6e532012a33c4c8a2a892bd1818: Waiting for close lock at 1733761860067Running coprocessor pre-close hooks at 1733761860067Disabling compacts and flushes for region at 1733761860067Disabling writes for close at 1733761860067Writing region close event to WAL at 1733761860069 (+2 ms)Running coprocessor post-close hooks at 1733761860069Closed at 1733761860070 (+1 ms) 2024-12-09T16:31:00,070 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:31:00,084 INFO [regionserver/dfce6ce5b10c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T16:31:00,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server dfce6ce5b10c,32855,1733761840798 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:31:00,181 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,32855,1733761840798, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,32855,1733761840798, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server dfce6ce5b10c,32855,1733761840798 aborting 2024-12-09T16:31:00,181 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,32855,1733761840798, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server dfce6ce5b10c,32855,1733761840798 aborting 2024-12-09T16:31:00,181 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,32855,1733761840798, seqNum=12 from cache 2024-12-09T16:31:00,233 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-09T16:31:00,234 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-09T16:31:00,235 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-09T16:31:00,235 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-09T16:31:00,269 INFO [RS:1;dfce6ce5b10c:32855 {}] regionserver.HRegionServer(976): stopping server dfce6ce5b10c,32855,1733761840798; all regions closed. 2024-12-09T16:31:00,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741834_1010 (size=1677) 2024-12-09T16:31:00,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741834_1010 (size=1677) 2024-12-09T16:31:00,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741834_1010 (size=1677) 2024-12-09T16:31:00,274 DEBUG [RS:1;dfce6ce5b10c:32855 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T16:31:00,274 INFO [RS:1;dfce6ce5b10c:32855 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T16:31:00,274 INFO [RS:1;dfce6ce5b10c:32855 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T16:31:00,275 INFO [RS:1;dfce6ce5b10c:32855 {}] hbase.ChoreService(370): Chore service for: regionserver/dfce6ce5b10c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-09T16:31:00,275 INFO [regionserver/dfce6ce5b10c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T16:31:00,275 INFO [RS:1;dfce6ce5b10c:32855 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T16:31:00,275 INFO [RS:1;dfce6ce5b10c:32855 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T16:31:00,275 INFO [RS:1;dfce6ce5b10c:32855 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T16:31:00,275 INFO [RS:1;dfce6ce5b10c:32855 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T16:31:00,275 INFO [RS:1;dfce6ce5b10c:32855 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32855 2024-12-09T16:31:00,392 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,32855,1733761840798, seqNum=18] 2024-12-09T16:31:00,394 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server dfce6ce5b10c:32855 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: dfce6ce5b10c/172.17.0.2:32855 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:31:00,394 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,32855,1733761840798, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,32855,1733761840798, seqNum=18, error=java.net.ConnectException: Call to address=dfce6ce5b10c:32855 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: dfce6ce5b10c/172.17.0.2:32855 2024-12-09T16:31:00,395 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,32855,1733761840798, seqNum=18 is java.net.ConnectException: Connection refused 2024-12-09T16:31:00,395 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,32855,1733761840798, seqNum=18 from cache 2024-12-09T16:31:00,395 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address dfce6ce5b10c:32855 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: dfce6ce5b10c/172.17.0.2:32855 2024-12-09T16:31:00,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T16:31:00,492 INFO [RS:1;dfce6ce5b10c:32855 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T16:31:00,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32855-0x100bfc42e970002, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dfce6ce5b10c,32855,1733761840798 2024-12-09T16:31:00,500 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dfce6ce5b10c,32855,1733761840798] 2024-12-09T16:31:00,508 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dfce6ce5b10c,32855,1733761840798 already deleted, retry=false 2024-12-09T16:31:00,509 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of dfce6ce5b10c,32855,1733761840798 on dfce6ce5b10c,35827,1733761839880 2024-12-09T16:31:00,513 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure dfce6ce5b10c,32855,1733761840798, splitWal=true, meta=false 2024-12-09T16:31:00,515 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for dfce6ce5b10c,32855,1733761840798 (carryingMeta=false) dfce6ce5b10c,32855,1733761840798/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@2879873a[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-09T16:31:00,516 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure dfce6ce5b10c,32855,1733761840798, splitWal=true, meta=false 2024-12-09T16:31:00,517 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): dfce6ce5b10c,32855,1733761840798 had 1 regions 2024-12-09T16:31:00,518 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure dfce6ce5b10c,32855,1733761840798, splitWal=true, meta=false, isMeta: false 2024-12-09T16:31:00,520 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,32855,1733761840798-splitting 2024-12-09T16:31:00,521 INFO [PEWorker-1 {}] master.SplitWALManager(105): dfce6ce5b10c,32855,1733761840798 WAL count=1, meta=false 2024-12-09T16:31:00,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure dfce6ce5b10c%2C32855%2C1733761840798.1733761842531}] 2024-12-09T16:31:00,530 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=dfce6ce5b10c,34709,1733761840861 2024-12-09T16:31:00,532 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure dfce6ce5b10c%2C32855%2C1733761840798.1733761842531, worker=dfce6ce5b10c,34709,1733761840861}] 2024-12-09T16:31:00,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32855-0x100bfc42e970002, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T16:31:00,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32855-0x100bfc42e970002, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T16:31:00,601 INFO [RS:1;dfce6ce5b10c:32855 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T16:31:00,601 INFO [RS:1;dfce6ce5b10c:32855 {}] regionserver.HRegionServer(1031): Exiting; stopping=dfce6ce5b10c,32855,1733761840798; zookeeper connection closed. 2024-12-09T16:31:00,602 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@33af36d7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@33af36d7 2024-12-09T16:31:00,696 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T16:31:00,697 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39617, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T16:31:00,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34709 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-09T16:31:00,706 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,32855,1733761840798, seqNum=18] 2024-12-09T16:31:00,706 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to dfce6ce5b10c:32855 this server is in the failed servers list 2024-12-09T16:31:00,707 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,32855,1733761840798, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,32855,1733761840798, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=dfce6ce5b10c:32855 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: dfce6ce5b10c:32855 2024-12-09T16:31:00,707 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,32855,1733761840798, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: dfce6ce5b10c:32855 2024-12-09T16:31:00,707 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,32855,1733761840798, seqNum=18 from cache 2024-12-09T16:31:00,718 INFO [RS_LOG_REPLAY_OPS-regionserver/dfce6ce5b10c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,32855,1733761840798-splitting/dfce6ce5b10c%2C32855%2C1733761840798.1733761842531, size=1.6 K (1677bytes) 2024-12-09T16:31:00,718 INFO [RS_LOG_REPLAY_OPS-regionserver/dfce6ce5b10c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,32855,1733761840798-splitting/dfce6ce5b10c%2C32855%2C1733761840798.1733761842531 2024-12-09T16:31:00,719 INFO [RS_LOG_REPLAY_OPS-regionserver/dfce6ce5b10c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,32855,1733761840798-splitting/dfce6ce5b10c%2C32855%2C1733761840798.1733761842531 after 1ms 2024-12-09T16:31:00,722 DEBUG [RS_LOG_REPLAY_OPS-regionserver/dfce6ce5b10c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,32855,1733761840798-splitting/dfce6ce5b10c%2C32855%2C1733761840798.1733761842531: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:31:00,722 INFO [RS_LOG_REPLAY_OPS-regionserver/dfce6ce5b10c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,32855,1733761840798-splitting/dfce6ce5b10c%2C32855%2C1733761840798.1733761842531 took 4ms 2024-12-09T16:31:00,732 DEBUG [RS_LOG_REPLAY_OPS-regionserver/dfce6ce5b10c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for f010e6e532012a33c4c8a2a892bd1818: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-09T16:31:00,732 DEBUG [RS_LOG_REPLAY_OPS-regionserver/dfce6ce5b10c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,32855,1733761840798-splitting/dfce6ce5b10c%2C32855%2C1733761840798.1733761842531 so closing down 2024-12-09T16:31:00,732 DEBUG [RS_LOG_REPLAY_OPS-regionserver/dfce6ce5b10c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T16:31:00,732 INFO [RS_LOG_REPLAY_OPS-regionserver/dfce6ce5b10c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T16:31:00,732 INFO [RS_LOG_REPLAY_OPS-regionserver/dfce6ce5b10c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 9 ms; skipped=6; WAL=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,32855,1733761840798-splitting/dfce6ce5b10c%2C32855%2C1733761840798.1733761842531, size=1.6 K, length=1677, corrupted=false, cancelled=false 2024-12-09T16:31:00,732 DEBUG [RS_LOG_REPLAY_OPS-regionserver/dfce6ce5b10c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,32855,1733761840798-splitting/dfce6ce5b10c%2C32855%2C1733761840798.1733761842531, journal: Splitting hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,32855,1733761840798-splitting/dfce6ce5b10c%2C32855%2C1733761840798.1733761842531, size=1.6 K (1677bytes) at 1733761860718Finishing writing output for hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,32855,1733761840798-splitting/dfce6ce5b10c%2C32855%2C1733761840798.1733761842531 so closing down at 1733761860732 (+14 ms)3 split writer threads finished at 1733761860732Processed 6 edits across 0 Regions in 9 ms; skipped=6; WAL=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,32855,1733761840798-splitting/dfce6ce5b10c%2C32855%2C1733761840798.1733761842531, size=1.6 K, length=1677, corrupted=false, cancelled=false at 1733761860732 2024-12-09T16:31:00,732 DEBUG [RS_LOG_REPLAY_OPS-regionserver/dfce6ce5b10c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,32855,1733761840798-splitting/dfce6ce5b10c%2C32855%2C1733761840798.1733761842531 2024-12-09T16:31:00,734 DEBUG [RS_LOG_REPLAY_OPS-regionserver/dfce6ce5b10c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-09T16:31:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35827 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-09T16:31:00,742 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,32855,1733761840798-splitting/dfce6ce5b10c%2C32855%2C1733761840798.1733761842531 to hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/oldWALs 2024-12-09T16:31:00,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-09T16:31:00,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure dfce6ce5b10c%2C32855%2C1733761840798.1733761842531, worker=dfce6ce5b10c,34709,1733761840861 in 210 msec 2024-12-09T16:31:00,747 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=dfce6ce5b10c,34709,1733761840861 2024-12-09T16:31:00,756 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-09T16:31:00,756 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure dfce6ce5b10c%2C32855%2C1733761840798.1733761842531, worker=dfce6ce5b10c,34709,1733761840861 in 225 msec 2024-12-09T16:31:00,758 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,32855,1733761840798-splitting dir is empty, no logs to split. 2024-12-09T16:31:00,758 INFO [PEWorker-1 {}] master.SplitWALManager(105): dfce6ce5b10c,32855,1733761840798 WAL count=0, meta=false 2024-12-09T16:31:00,758 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if dfce6ce5b10c,32855,1733761840798 WAL splitting is done? wals=0, meta=false 2024-12-09T16:31:00,760 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for dfce6ce5b10c,32855,1733761840798 failed, ignore...File hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/WALs/dfce6ce5b10c,32855,1733761840798-splitting does not exist. 2024-12-09T16:31:00,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f010e6e532012a33c4c8a2a892bd1818, ASSIGN}] 2024-12-09T16:31:00,765 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f010e6e532012a33c4c8a2a892bd1818, ASSIGN 2024-12-09T16:31:00,766 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f010e6e532012a33c4c8a2a892bd1818, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-09T16:31:00,917 DEBUG [dfce6ce5b10c:35827 {}] balancer.BalancerClusterState(204): Hosts are {dfce6ce5b10c=0} racks are {/default-rack=0} 2024-12-09T16:31:00,917 DEBUG [dfce6ce5b10c:35827 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T16:31:00,917 DEBUG [dfce6ce5b10c:35827 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T16:31:00,918 DEBUG [dfce6ce5b10c:35827 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T16:31:00,918 DEBUG [dfce6ce5b10c:35827 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T16:31:00,918 INFO [dfce6ce5b10c:35827 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T16:31:00,918 INFO [dfce6ce5b10c:35827 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T16:31:00,918 DEBUG [dfce6ce5b10c:35827 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T16:31:00,918 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=f010e6e532012a33c4c8a2a892bd1818, regionState=OPENING, regionLocation=dfce6ce5b10c,34709,1733761840861 2024-12-09T16:31:00,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f010e6e532012a33c4c8a2a892bd1818, ASSIGN because future has completed 2024-12-09T16:31:00,923 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure f010e6e532012a33c4c8a2a892bd1818, server=dfce6ce5b10c,34709,1733761840861}] 2024-12-09T16:31:01,083 INFO [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:31:01,083 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => f010e6e532012a33c4c8a2a892bd1818, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818.', STARTKEY => '', ENDKEY => ''} 2024-12-09T16:31:01,084 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:31:01,084 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:31:01,084 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:31:01,084 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:31:01,086 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:31:01,088 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f010e6e532012a33c4c8a2a892bd1818 columnFamilyName cf1 2024-12-09T16:31:01,088 DEBUG [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:01,097 DEBUG [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf1/e72135fa8984470d855ea27665b50e2b 2024-12-09T16:31:01,097 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.HStore(327): Store=f010e6e532012a33c4c8a2a892bd1818/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:01,097 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:31:01,098 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f010e6e532012a33c4c8a2a892bd1818 columnFamilyName cf2 2024-12-09T16:31:01,098 DEBUG [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:01,105 DEBUG [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/cf2/f5c013cde7ad4c8c86a9a6dc59d717d9 2024-12-09T16:31:01,105 INFO [StoreOpener-f010e6e532012a33c4c8a2a892bd1818-1 {}] regionserver.HStore(327): Store=f010e6e532012a33c4c8a2a892bd1818/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:01,105 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:31:01,107 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:31:01,108 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:31:01,109 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:31:01,109 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:31:01,109 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T16:31:01,111 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:31:01,112 INFO [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened f010e6e532012a33c4c8a2a892bd1818; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70320808, jitterRate=0.047861695289611816}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T16:31:01,112 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:31:01,113 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for f010e6e532012a33c4c8a2a892bd1818: Running coprocessor pre-open hook at 1733761861084Writing region info on filesystem at 1733761861084Initializing all the Stores at 1733761861085 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761861086 (+1 ms)Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761861086Cleaning up temporary data from old regions at 1733761861109 (+23 ms)Running coprocessor post-open hooks at 1733761861112 (+3 ms)Region opened successfully at 1733761861113 (+1 ms) 2024-12-09T16:31:01,114 INFO [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., pid=17, masterSystemTime=1733761861076 2024-12-09T16:31:01,116 DEBUG [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:31:01,117 INFO [RS_OPEN_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:31:01,118 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=f010e6e532012a33c4c8a2a892bd1818, regionState=OPEN, openSeqNum=18, regionLocation=dfce6ce5b10c,34709,1733761840861 2024-12-09T16:31:01,120 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure f010e6e532012a33c4c8a2a892bd1818, server=dfce6ce5b10c,34709,1733761840861 because future has completed 2024-12-09T16:31:01,124 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-09T16:31:01,124 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure f010e6e532012a33c4c8a2a892bd1818, server=dfce6ce5b10c,34709,1733761840861 in 198 msec 2024-12-09T16:31:01,127 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-09T16:31:01,127 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server dfce6ce5b10c,32855,1733761840798 after splitting done 2024-12-09T16:31:01,127 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=f010e6e532012a33c4c8a2a892bd1818, ASSIGN in 361 msec 2024-12-09T16:31:01,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure dfce6ce5b10c,32855,1733761840798, splitWal=true, meta=false in 618 msec 2024-12-09T16:31:01,222 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818., hostname=dfce6ce5b10c,34709,1733761840861, seqNum=18] 2024-12-09T16:31:01,223 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T16:31:01,225 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T16:31:01,242 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=411 (was 418), OpenFileDescriptor=1048 (was 1025) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=609 (was 636), ProcessCount=11 (was 11), AvailableMemoryMB=8888 (was 6434) - AvailableMemoryMB LEAK? - 2024-12-09T16:31:01,242 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-12-09T16:31:01,258 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=411, OpenFileDescriptor=1048, MaxFileDescriptor=1048576, SystemLoadAverage=609, ProcessCount=11, AvailableMemoryMB=8887 2024-12-09T16:31:01,258 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-12-09T16:31:01,273 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:31:01,276 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:31:01,277 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T16:31:01,280 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-89208068, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/hregion-89208068, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:31:01,295 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-89208068/hregion-89208068.1733761861280, exclude list is [], retry=0 2024-12-09T16:31:01,298 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:31:01,298 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:31:01,301 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:31:01,308 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-89208068/hregion-89208068.1733761861280 2024-12-09T16:31:01,309 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39661:39661),(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:35519:35519)] 2024-12-09T16:31:01,309 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ddd954b63b6213d1a20dc5e0cc810bba, NAME => 'testReplayEditsWrittenViaHRegion,,1733761861274.ddd954b63b6213d1a20dc5e0cc810bba.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39607/hbase 2024-12-09T16:31:01,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741893_1071 (size=67) 2024-12-09T16:31:01,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741893_1071 (size=67) 2024-12-09T16:31:01,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741893_1071 (size=67) 2024-12-09T16:31:01,720 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733761861274.ddd954b63b6213d1a20dc5e0cc810bba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:31:01,721 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:01,723 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ddd954b63b6213d1a20dc5e0cc810bba columnFamilyName a 2024-12-09T16:31:01,723 DEBUG [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:01,723 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] regionserver.HStore(327): Store=ddd954b63b6213d1a20dc5e0cc810bba/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:01,724 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:01,725 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ddd954b63b6213d1a20dc5e0cc810bba columnFamilyName b 2024-12-09T16:31:01,725 DEBUG [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:01,726 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] regionserver.HStore(327): Store=ddd954b63b6213d1a20dc5e0cc810bba/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:01,726 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:01,728 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ddd954b63b6213d1a20dc5e0cc810bba columnFamilyName c 2024-12-09T16:31:01,728 DEBUG [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:01,730 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] regionserver.HStore(327): Store=ddd954b63b6213d1a20dc5e0cc810bba/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:01,730 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:01,731 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:01,731 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:01,732 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:01,732 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:01,733 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T16:31:01,735 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:01,737 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T16:31:01,737 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ddd954b63b6213d1a20dc5e0cc810bba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62888666, jitterRate=-0.06288585066795349}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T16:31:01,738 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ddd954b63b6213d1a20dc5e0cc810bba: Writing region info on filesystem at 1733761861720Initializing all the Stores at 1733761861721 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761861721Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761861721Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761861721Cleaning up temporary data from old regions at 1733761861732 (+11 ms)Region opened successfully at 1733761861738 (+6 ms) 2024-12-09T16:31:01,738 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ddd954b63b6213d1a20dc5e0cc810bba, disabling compactions & flushes 2024-12-09T16:31:01,739 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733761861274.ddd954b63b6213d1a20dc5e0cc810bba. 2024-12-09T16:31:01,739 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733761861274.ddd954b63b6213d1a20dc5e0cc810bba. 2024-12-09T16:31:01,739 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733761861274.ddd954b63b6213d1a20dc5e0cc810bba. after waiting 0 ms 2024-12-09T16:31:01,739 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733761861274.ddd954b63b6213d1a20dc5e0cc810bba. 2024-12-09T16:31:01,740 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733761861274.ddd954b63b6213d1a20dc5e0cc810bba. 2024-12-09T16:31:01,740 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ddd954b63b6213d1a20dc5e0cc810bba: Waiting for close lock at 1733761861738Disabling compacts and flushes for region at 1733761861738Disabling writes for close at 1733761861739 (+1 ms)Writing region close event to WAL at 1733761861740 (+1 ms)Closed at 1733761861740 2024-12-09T16:31:01,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741892_1070 (size=93) 2024-12-09T16:31:01,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741892_1070 (size=93) 2024-12-09T16:31:01,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741892_1070 (size=93) 2024-12-09T16:31:01,746 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T16:31:01,747 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-89208068:(num 1733761861280) 2024-12-09T16:31:01,747 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T16:31:01,749 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733761861273, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:31:01,764 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733761861273/wal.1733761861750, exclude list is [], retry=0 2024-12-09T16:31:01,768 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:31:01,768 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:31:01,768 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:31:01,770 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733761861273/wal.1733761861750 2024-12-09T16:31:01,770 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:39661:39661),(127.0.0.1/127.0.0.1:35519:35519)] 2024-12-09T16:31:01,771 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ddd954b63b6213d1a20dc5e0cc810bba, NAME => 'testReplayEditsWrittenViaHRegion,,1733761861274.ddd954b63b6213d1a20dc5e0cc810bba.', STARTKEY => '', ENDKEY => ''} 2024-12-09T16:31:01,771 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733761861274.ddd954b63b6213d1a20dc5e0cc810bba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:31:01,771 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:01,771 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:01,772 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:01,773 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ddd954b63b6213d1a20dc5e0cc810bba columnFamilyName a 2024-12-09T16:31:01,773 DEBUG [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:01,774 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] regionserver.HStore(327): Store=ddd954b63b6213d1a20dc5e0cc810bba/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:01,774 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:01,775 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ddd954b63b6213d1a20dc5e0cc810bba columnFamilyName b 2024-12-09T16:31:01,775 DEBUG [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:01,775 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] regionserver.HStore(327): Store=ddd954b63b6213d1a20dc5e0cc810bba/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:01,775 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:01,776 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ddd954b63b6213d1a20dc5e0cc810bba columnFamilyName c 2024-12-09T16:31:01,776 DEBUG [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:01,777 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] regionserver.HStore(327): Store=ddd954b63b6213d1a20dc5e0cc810bba/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:01,777 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:01,778 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:01,780 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:01,781 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:01,781 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:01,782 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T16:31:01,784 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:01,785 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ddd954b63b6213d1a20dc5e0cc810bba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67837753, jitterRate=0.010861292481422424}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T16:31:01,787 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ddd954b63b6213d1a20dc5e0cc810bba: Writing region info on filesystem at 1733761861771Initializing all the Stores at 1733761861772 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761861772Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761861772Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761861772Cleaning up temporary data from old regions at 1733761861781 (+9 ms)Region opened successfully at 1733761861786 (+5 ms) 2024-12-09T16:31:01,818 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ddd954b63b6213d1a20dc5e0cc810bba 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-09T16:31:01,839 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/.tmp/a/453900d6d6e24905870ce478a0dfaa8e is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733761861787/Put/seqid=0 2024-12-09T16:31:01,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741895_1073 (size=5958) 2024-12-09T16:31:01,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741895_1073 (size=5958) 2024-12-09T16:31:01,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741895_1073 (size=5958) 2024-12-09T16:31:01,850 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/.tmp/a/453900d6d6e24905870ce478a0dfaa8e 2024-12-09T16:31:01,875 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/.tmp/b/bdf0cceb23ac4374bb04f444b46d47b5 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733761861797/Put/seqid=0 2024-12-09T16:31:01,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741896_1074 (size=5958) 2024-12-09T16:31:01,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741896_1074 (size=5958) 2024-12-09T16:31:01,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741896_1074 (size=5958) 2024-12-09T16:31:01,884 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/.tmp/b/bdf0cceb23ac4374bb04f444b46d47b5 2024-12-09T16:31:01,911 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/.tmp/c/087fbe3f99c14bbbb910b89af7ca3320 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733761861806/Put/seqid=0 2024-12-09T16:31:01,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741897_1075 (size=5958) 2024-12-09T16:31:01,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741897_1075 (size=5958) 2024-12-09T16:31:01,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741897_1075 (size=5958) 2024-12-09T16:31:01,918 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/.tmp/c/087fbe3f99c14bbbb910b89af7ca3320 2024-12-09T16:31:01,927 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/.tmp/a/453900d6d6e24905870ce478a0dfaa8e as hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/a/453900d6d6e24905870ce478a0dfaa8e 2024-12-09T16:31:01,936 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/a/453900d6d6e24905870ce478a0dfaa8e, entries=10, sequenceid=33, filesize=5.8 K 2024-12-09T16:31:01,937 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/.tmp/b/bdf0cceb23ac4374bb04f444b46d47b5 as hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/b/bdf0cceb23ac4374bb04f444b46d47b5 2024-12-09T16:31:01,946 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/b/bdf0cceb23ac4374bb04f444b46d47b5, entries=10, sequenceid=33, filesize=5.8 K 2024-12-09T16:31:01,947 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/.tmp/c/087fbe3f99c14bbbb910b89af7ca3320 as hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/c/087fbe3f99c14bbbb910b89af7ca3320 2024-12-09T16:31:01,953 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/c/087fbe3f99c14bbbb910b89af7ca3320, entries=10, sequenceid=33, filesize=5.8 K 2024-12-09T16:31:01,955 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for ddd954b63b6213d1a20dc5e0cc810bba in 138ms, sequenceid=33, compaction requested=false 2024-12-09T16:31:01,955 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ddd954b63b6213d1a20dc5e0cc810bba: 2024-12-09T16:31:01,955 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ddd954b63b6213d1a20dc5e0cc810bba, disabling compactions & flushes 2024-12-09T16:31:01,956 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733761861274.ddd954b63b6213d1a20dc5e0cc810bba. 2024-12-09T16:31:01,956 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733761861274.ddd954b63b6213d1a20dc5e0cc810bba. 2024-12-09T16:31:01,956 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733761861274.ddd954b63b6213d1a20dc5e0cc810bba. after waiting 0 ms 2024-12-09T16:31:01,956 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733761861274.ddd954b63b6213d1a20dc5e0cc810bba. 2024-12-09T16:31:01,957 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733761861274.ddd954b63b6213d1a20dc5e0cc810bba. 2024-12-09T16:31:01,957 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ddd954b63b6213d1a20dc5e0cc810bba: Waiting for close lock at 1733761861955Disabling compacts and flushes for region at 1733761861955Disabling writes for close at 1733761861956 (+1 ms)Writing region close event to WAL at 1733761861957 (+1 ms)Closed at 1733761861957 2024-12-09T16:31:01,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741894_1072 (size=2873) 2024-12-09T16:31:01,960 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733761861273/wal.1733761861750 not finished, retry = 0 2024-12-09T16:31:01,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741894_1072 (size=2873) 2024-12-09T16:31:01,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741894_1072 (size=2873) 2024-12-09T16:31:02,066 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/b/bdf0cceb23ac4374bb04f444b46d47b5 to hdfs://localhost:39607/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/b/bdf0cceb23ac4374bb04f444b46d47b5 2024-12-09T16:31:02,084 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39607/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733761861273/wal.1733761861750, size=2.8 K (2873bytes) 2024-12-09T16:31:02,084 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39607/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733761861273/wal.1733761861750 2024-12-09T16:31:02,084 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39607/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733761861273/wal.1733761861750 after 0ms 2024-12-09T16:31:02,087 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733761861273/wal.1733761861750: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:31:02,087 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39607/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733761861273/wal.1733761861750 took 3ms 2024-12-09T16:31:02,090 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39607/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733761861273/wal.1733761861750 so closing down 2024-12-09T16:31:02,090 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T16:31:02,091 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733761861750.temp 2024-12-09T16:31:02,092 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/recovered.edits/0000000000000000003-wal.1733761861750.temp 2024-12-09T16:31:02,092 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T16:31:02,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741898_1076 (size=2312) 2024-12-09T16:31:02,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741898_1076 (size=2312) 2024-12-09T16:31:02,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741898_1076 (size=2312) 2024-12-09T16:31:02,101 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/recovered.edits/0000000000000000003-wal.1733761861750.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-09T16:31:02,102 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/recovered.edits/0000000000000000003-wal.1733761861750.temp to hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/recovered.edits/0000000000000000032 2024-12-09T16:31:02,102 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:39607/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733761861273/wal.1733761861750, size=2.8 K, length=2873, corrupted=false, cancelled=false 2024-12-09T16:31:02,102 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39607/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733761861273/wal.1733761861750, journal: Splitting hdfs://localhost:39607/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733761861273/wal.1733761861750, size=2.8 K (2873bytes) at 1733761862084Finishing writing output for hdfs://localhost:39607/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733761861273/wal.1733761861750 so closing down at 1733761862090 (+6 ms)Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/recovered.edits/0000000000000000003-wal.1733761861750.temp at 1733761862092 (+2 ms)3 split writer threads finished at 1733761862092Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/recovered.edits/0000000000000000003-wal.1733761861750.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733761862101 (+9 ms)Rename recovered edits hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/recovered.edits/0000000000000000003-wal.1733761861750.temp to hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/recovered.edits/0000000000000000032 at 1733761862102 (+1 ms)Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:39607/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733761861273/wal.1733761861750, size=2.8 K, length=2873, corrupted=false, cancelled=false at 1733761862102 2024-12-09T16:31:02,104 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39607/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733761861273/wal.1733761861750 to hdfs://localhost:39607/hbase/oldWALs/wal.1733761861750 2024-12-09T16:31:02,105 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/recovered.edits/0000000000000000032 2024-12-09T16:31:02,105 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T16:31:02,107 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733761861273, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:31:02,121 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733761861273/wal.1733761862107, exclude list is [], retry=0 2024-12-09T16:31:02,124 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:31:02,125 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:31:02,125 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:31:02,130 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733761861273/wal.1733761862107 2024-12-09T16:31:02,131 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:39661:39661),(127.0.0.1/127.0.0.1:37099:37099)] 2024-12-09T16:31:02,131 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ddd954b63b6213d1a20dc5e0cc810bba, NAME => 'testReplayEditsWrittenViaHRegion,,1733761861274.ddd954b63b6213d1a20dc5e0cc810bba.', STARTKEY => '', ENDKEY => ''} 2024-12-09T16:31:02,131 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733761861274.ddd954b63b6213d1a20dc5e0cc810bba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:31:02,131 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:02,131 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:02,133 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:02,134 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ddd954b63b6213d1a20dc5e0cc810bba columnFamilyName a 2024-12-09T16:31:02,134 DEBUG [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:02,141 DEBUG [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/a/453900d6d6e24905870ce478a0dfaa8e 2024-12-09T16:31:02,141 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] regionserver.HStore(327): Store=ddd954b63b6213d1a20dc5e0cc810bba/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:02,141 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:02,143 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ddd954b63b6213d1a20dc5e0cc810bba columnFamilyName b 2024-12-09T16:31:02,143 DEBUG [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:02,144 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] regionserver.HStore(327): Store=ddd954b63b6213d1a20dc5e0cc810bba/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:02,144 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:02,144 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ddd954b63b6213d1a20dc5e0cc810bba columnFamilyName c 2024-12-09T16:31:02,145 DEBUG [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:02,155 DEBUG [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/c/087fbe3f99c14bbbb910b89af7ca3320 2024-12-09T16:31:02,155 INFO [StoreOpener-ddd954b63b6213d1a20dc5e0cc810bba-1 {}] regionserver.HStore(327): Store=ddd954b63b6213d1a20dc5e0cc810bba/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:02,155 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:02,157 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:02,159 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:02,160 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/recovered.edits/0000000000000000032 2024-12-09T16:31:02,163 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:31:02,164 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/recovered.edits/0000000000000000032 2024-12-09T16:31:02,164 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ddd954b63b6213d1a20dc5e0cc810bba 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-09T16:31:02,187 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/.tmp/b/2502ede630cd4de589cba56f7ad0d255 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733761861797/Put/seqid=0 2024-12-09T16:31:02,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741900_1078 (size=5958) 2024-12-09T16:31:02,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741900_1078 (size=5958) 2024-12-09T16:31:02,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741900_1078 (size=5958) 2024-12-09T16:31:02,198 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/.tmp/b/2502ede630cd4de589cba56f7ad0d255 2024-12-09T16:31:02,207 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/.tmp/b/2502ede630cd4de589cba56f7ad0d255 as hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/b/2502ede630cd4de589cba56f7ad0d255 2024-12-09T16:31:02,214 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/b/2502ede630cd4de589cba56f7ad0d255, entries=10, sequenceid=32, filesize=5.8 K 2024-12-09T16:31:02,214 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for ddd954b63b6213d1a20dc5e0cc810bba in 50ms, sequenceid=32, compaction requested=false; wal=null 2024-12-09T16:31:02,215 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/recovered.edits/0000000000000000032 2024-12-09T16:31:02,216 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:02,216 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:02,217 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T16:31:02,219 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ddd954b63b6213d1a20dc5e0cc810bba 2024-12-09T16:31:02,221 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/ddd954b63b6213d1a20dc5e0cc810bba/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-09T16:31:02,222 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ddd954b63b6213d1a20dc5e0cc810bba; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73413563, jitterRate=0.09394733607769012}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T16:31:02,223 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ddd954b63b6213d1a20dc5e0cc810bba: Writing region info on filesystem at 1733761862131Initializing all the Stores at 1733761862132 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761862132Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761862133 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761862133Obtaining lock to block concurrent updates at 1733761862164 (+31 ms)Preparing flush snapshotting stores in ddd954b63b6213d1a20dc5e0cc810bba at 1733761862164Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733761861274.ddd954b63b6213d1a20dc5e0cc810bba., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733761862165 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1733761861274.ddd954b63b6213d1a20dc5e0cc810bba. at 1733761862165Flushing ddd954b63b6213d1a20dc5e0cc810bba/b: creating writer at 1733761862165Flushing ddd954b63b6213d1a20dc5e0cc810bba/b: appending metadata at 1733761862186 (+21 ms)Flushing ddd954b63b6213d1a20dc5e0cc810bba/b: closing flushed file at 1733761862187 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d3c3196: reopening flushed file at 1733761862206 (+19 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for ddd954b63b6213d1a20dc5e0cc810bba in 50ms, sequenceid=32, compaction requested=false; wal=null at 1733761862214 (+8 ms)Cleaning up temporary data from old regions at 1733761862216 (+2 ms)Region opened successfully at 1733761862223 (+7 ms) 2024-12-09T16:31:02,246 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=420 (was 411) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:33742 [Waiting for operation #44] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-427817209-172.17.0.2-1733761835089:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:56052 [Waiting for operation #28] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-427817209-172.17.0.2-1733761835089:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:56620 [Waiting for operation #30] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:56200 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:56768 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:33896 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-427817209-172.17.0.2-1733761835089:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1124 (was 1048) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=609 (was 609), ProcessCount=11 (was 11), AvailableMemoryMB=8871 (was 8887) 2024-12-09T16:31:02,246 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-12-09T16:31:02,258 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=420, OpenFileDescriptor=1124, MaxFileDescriptor=1048576, SystemLoadAverage=609, ProcessCount=11, AvailableMemoryMB=8869 2024-12-09T16:31:02,258 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-12-09T16:31:02,275 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:31:02,278 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:31:02,279 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T16:31:02,282 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-45500463, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/hregion-45500463, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:31:02,296 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-45500463/hregion-45500463.1733761862282, exclude list is [], retry=0 2024-12-09T16:31:02,301 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:31:02,301 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:31:02,302 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:31:02,306 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-45500463/hregion-45500463.1733761862282 2024-12-09T16:31:02,306 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:39661:39661)] 2024-12-09T16:31:02,307 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 327eab9f241eb256cde10340b0ae99f8, NAME => 'testReplayEditsAfterAbortingFlush,,1733761862276.327eab9f241eb256cde10340b0ae99f8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39607/hbase 2024-12-09T16:31:02,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741902_1080 (size=68) 2024-12-09T16:31:02,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741902_1080 (size=68) 2024-12-09T16:31:02,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741902_1080 (size=68) 2024-12-09T16:31:02,321 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733761862276.327eab9f241eb256cde10340b0ae99f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:31:02,323 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:02,324 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 327eab9f241eb256cde10340b0ae99f8 columnFamilyName a 2024-12-09T16:31:02,324 DEBUG [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:02,325 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] regionserver.HStore(327): Store=327eab9f241eb256cde10340b0ae99f8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:02,325 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:02,328 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 327eab9f241eb256cde10340b0ae99f8 columnFamilyName b 2024-12-09T16:31:02,328 DEBUG [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:02,329 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] regionserver.HStore(327): Store=327eab9f241eb256cde10340b0ae99f8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:02,329 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:02,330 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 327eab9f241eb256cde10340b0ae99f8 columnFamilyName c 2024-12-09T16:31:02,331 DEBUG [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:02,331 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] regionserver.HStore(327): Store=327eab9f241eb256cde10340b0ae99f8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:02,331 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:02,332 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:02,332 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:02,333 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:02,333 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:02,334 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T16:31:02,335 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:02,337 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T16:31:02,338 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 327eab9f241eb256cde10340b0ae99f8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69503444, jitterRate=0.03568202257156372}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T16:31:02,339 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 327eab9f241eb256cde10340b0ae99f8: Writing region info on filesystem at 1733761862321Initializing all the Stores at 1733761862322 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761862322Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761862323 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761862323Cleaning up temporary data from old regions at 1733761862333 (+10 ms)Region opened successfully at 1733761862339 (+6 ms) 2024-12-09T16:31:02,339 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 327eab9f241eb256cde10340b0ae99f8, disabling compactions & flushes 2024-12-09T16:31:02,339 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733761862276.327eab9f241eb256cde10340b0ae99f8. 2024-12-09T16:31:02,339 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733761862276.327eab9f241eb256cde10340b0ae99f8. 2024-12-09T16:31:02,339 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733761862276.327eab9f241eb256cde10340b0ae99f8. after waiting 0 ms 2024-12-09T16:31:02,340 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733761862276.327eab9f241eb256cde10340b0ae99f8. 2024-12-09T16:31:02,340 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733761862276.327eab9f241eb256cde10340b0ae99f8. 2024-12-09T16:31:02,340 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 327eab9f241eb256cde10340b0ae99f8: Waiting for close lock at 1733761862339Disabling compacts and flushes for region at 1733761862339Disabling writes for close at 1733761862340 (+1 ms)Writing region close event to WAL at 1733761862340Closed at 1733761862340 2024-12-09T16:31:02,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741901_1079 (size=93) 2024-12-09T16:31:02,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741901_1079 (size=93) 2024-12-09T16:31:02,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741901_1079 (size=93) 2024-12-09T16:31:02,347 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T16:31:02,347 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-45500463:(num 1733761862282) 2024-12-09T16:31:02,347 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T16:31:02,349 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733761862274, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:31:02,368 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733761862274/wal.1733761862350, exclude list is [], retry=0 2024-12-09T16:31:02,371 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:31:02,371 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:31:02,372 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:31:02,373 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733761862274/wal.1733761862350 2024-12-09T16:31:02,374 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:39661:39661),(127.0.0.1/127.0.0.1:37099:37099)] 2024-12-09T16:31:02,450 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 327eab9f241eb256cde10340b0ae99f8, NAME => 'testReplayEditsAfterAbortingFlush,,1733761862276.327eab9f241eb256cde10340b0ae99f8.', STARTKEY => '', ENDKEY => ''} 2024-12-09T16:31:02,452 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:02,452 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733761862276.327eab9f241eb256cde10340b0ae99f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:31:02,453 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:02,453 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:02,455 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:02,456 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 327eab9f241eb256cde10340b0ae99f8 columnFamilyName a 2024-12-09T16:31:02,456 DEBUG [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:02,457 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] regionserver.HStore(327): Store=327eab9f241eb256cde10340b0ae99f8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:02,457 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:02,458 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 327eab9f241eb256cde10340b0ae99f8 columnFamilyName b 2024-12-09T16:31:02,458 DEBUG [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:02,459 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] regionserver.HStore(327): Store=327eab9f241eb256cde10340b0ae99f8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:02,459 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:02,460 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 327eab9f241eb256cde10340b0ae99f8 columnFamilyName c 2024-12-09T16:31:02,460 DEBUG [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:02,460 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] regionserver.HStore(327): Store=327eab9f241eb256cde10340b0ae99f8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:02,461 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:02,461 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:02,463 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:02,464 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:02,464 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:02,465 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T16:31:02,467 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:02,468 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 327eab9f241eb256cde10340b0ae99f8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63616074, jitterRate=-0.052046626806259155}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T16:31:02,468 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:02,469 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 327eab9f241eb256cde10340b0ae99f8: Running coprocessor pre-open hook at 1733761862453Writing region info on filesystem at 1733761862453Initializing all the Stores at 1733761862454 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761862454Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761862454Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761862454Cleaning up temporary data from old regions at 1733761862464 (+10 ms)Running coprocessor post-open hooks at 1733761862468 (+4 ms)Region opened successfully at 1733761862469 (+1 ms) 2024-12-09T16:31:02,487 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 327eab9f241eb256cde10340b0ae99f8 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-09T16:31:02,489 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 327eab9f241eb256cde10340b0ae99f8/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:31:03,490 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 327eab9f241eb256cde10340b0ae99f8/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:31:03,636 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T16:31:04,491 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 327eab9f241eb256cde10340b0ae99f8/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:31:05,492 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 327eab9f241eb256cde10340b0ae99f8/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:31:06,492 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 327eab9f241eb256cde10340b0ae99f8/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:31:07,493 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 327eab9f241eb256cde10340b0ae99f8/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:31:08,493 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 327eab9f241eb256cde10340b0ae99f8/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:31:08,992 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T16:31:09,494 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 327eab9f241eb256cde10340b0ae99f8/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:31:10,233 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-09T16:31:10,234 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-09T16:31:10,235 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-09T16:31:10,235 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-09T16:31:10,236 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-09T16:31:10,236 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-09T16:31:10,495 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 327eab9f241eb256cde10340b0ae99f8/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:31:11,496 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 327eab9f241eb256cde10340b0ae99f8/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:31:11,497 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 327eab9f241eb256cde10340b0ae99f8: 2024-12-09T16:31:11,497 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733761862276.327eab9f241eb256cde10340b0ae99f8. 2024-12-09T16:31:11,520 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 327eab9f241eb256cde10340b0ae99f8: 2024-12-09T16:31:11,520 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-09T16:31:11,520 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 327eab9f241eb256cde10340b0ae99f8, disabling compactions & flushes 2024-12-09T16:31:11,520 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733761862276.327eab9f241eb256cde10340b0ae99f8. 2024-12-09T16:31:11,520 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733761862276.327eab9f241eb256cde10340b0ae99f8. 2024-12-09T16:31:11,520 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733761862276.327eab9f241eb256cde10340b0ae99f8. after waiting 0 ms 2024-12-09T16:31:11,520 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733761862276.327eab9f241eb256cde10340b0ae99f8. 2024-12-09T16:31:11,521 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733761862276.327eab9f241eb256cde10340b0ae99f8. 2024-12-09T16:31:11,521 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733761862276.327eab9f241eb256cde10340b0ae99f8. 2024-12-09T16:31:11,521 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 327eab9f241eb256cde10340b0ae99f8: Waiting for close lock at 1733761871520Running coprocessor pre-close hooks at 1733761871520Disabling compacts and flushes for region at 1733761871520Disabling writes for close at 1733761871520Writing region close event to WAL at 1733761871521 (+1 ms)Running coprocessor post-close hooks at 1733761871521Closed at 1733761871521 2024-12-09T16:31:11,525 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733761862274/wal.1733761862350 not finished, retry = 0 2024-12-09T16:31:11,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741903_1081 (size=2398) 2024-12-09T16:31:11,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741903_1081 (size=2398) 2024-12-09T16:31:11,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741903_1081 (size=2398) 2024-12-09T16:31:11,651 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39607/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733761862274/wal.1733761862350, size=2.3 K (2398bytes) 2024-12-09T16:31:11,651 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39607/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733761862274/wal.1733761862350 2024-12-09T16:31:11,652 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39607/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733761862274/wal.1733761862350 after 1ms 2024-12-09T16:31:11,655 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733761862274/wal.1733761862350: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:31:11,656 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39607/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733761862274/wal.1733761862350 took 5ms 2024-12-09T16:31:11,661 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39607/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733761862274/wal.1733761862350 so closing down 2024-12-09T16:31:11,661 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T16:31:11,663 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733761862350.temp 2024-12-09T16:31:11,671 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/recovered.edits/0000000000000000004-wal.1733761862350.temp 2024-12-09T16:31:11,671 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T16:31:11,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741904_1082 (size=1672) 2024-12-09T16:31:11,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741904_1082 (size=1672) 2024-12-09T16:31:11,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741904_1082 (size=1672) 2024-12-09T16:31:12,103 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/recovered.edits/0000000000000000004-wal.1733761862350.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-09T16:31:12,106 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/recovered.edits/0000000000000000004-wal.1733761862350.temp to hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/recovered.edits/0000000000000000026 2024-12-09T16:31:12,106 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 448 ms; skipped=3; WAL=hdfs://localhost:39607/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733761862274/wal.1733761862350, size=2.3 K, length=2398, corrupted=false, cancelled=false 2024-12-09T16:31:12,106 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39607/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733761862274/wal.1733761862350, journal: Splitting hdfs://localhost:39607/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733761862274/wal.1733761862350, size=2.3 K (2398bytes) at 1733761871651Finishing writing output for hdfs://localhost:39607/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733761862274/wal.1733761862350 so closing down at 1733761871661 (+10 ms)Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/recovered.edits/0000000000000000004-wal.1733761862350.temp at 1733761871671 (+10 ms)3 split writer threads finished at 1733761871671Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/recovered.edits/0000000000000000004-wal.1733761862350.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733761872104 (+433 ms)Rename recovered edits hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/recovered.edits/0000000000000000004-wal.1733761862350.temp to hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/recovered.edits/0000000000000000026 at 1733761872106 (+2 ms)Processed 23 edits across 1 Regions in 448 ms; skipped=3; WAL=hdfs://localhost:39607/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733761862274/wal.1733761862350, size=2.3 K, length=2398, corrupted=false, cancelled=false at 1733761872106 2024-12-09T16:31:12,108 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39607/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733761862274/wal.1733761862350 to hdfs://localhost:39607/hbase/oldWALs/wal.1733761862350 2024-12-09T16:31:12,110 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/recovered.edits/0000000000000000026 2024-12-09T16:31:12,110 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T16:31:12,113 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733761862274, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:31:12,127 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733761862274/wal.1733761872114, exclude list is [], retry=0 2024-12-09T16:31:12,130 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:31:12,130 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:31:12,131 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:31:12,140 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733761862274/wal.1733761872114 2024-12-09T16:31:12,140 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:39661:39661),(127.0.0.1/127.0.0.1:37099:37099)] 2024-12-09T16:31:12,141 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 327eab9f241eb256cde10340b0ae99f8, NAME => 'testReplayEditsAfterAbortingFlush,,1733761862276.327eab9f241eb256cde10340b0ae99f8.', STARTKEY => '', ENDKEY => ''} 2024-12-09T16:31:12,142 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:12,142 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733761862276.327eab9f241eb256cde10340b0ae99f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:31:12,142 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:12,142 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:12,145 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:12,147 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 327eab9f241eb256cde10340b0ae99f8 columnFamilyName a 2024-12-09T16:31:12,147 DEBUG [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:12,149 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] regionserver.HStore(327): Store=327eab9f241eb256cde10340b0ae99f8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:12,149 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:12,151 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 327eab9f241eb256cde10340b0ae99f8 columnFamilyName b 2024-12-09T16:31:12,151 DEBUG [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:12,152 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] regionserver.HStore(327): Store=327eab9f241eb256cde10340b0ae99f8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:12,153 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:12,154 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 327eab9f241eb256cde10340b0ae99f8 columnFamilyName c 2024-12-09T16:31:12,154 DEBUG [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:12,155 INFO [StoreOpener-327eab9f241eb256cde10340b0ae99f8-1 {}] regionserver.HStore(327): Store=327eab9f241eb256cde10340b0ae99f8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:12,155 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:12,156 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:12,159 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:12,160 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/recovered.edits/0000000000000000026 2024-12-09T16:31:12,167 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:31:12,171 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/recovered.edits/0000000000000000026 2024-12-09T16:31:12,171 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 327eab9f241eb256cde10340b0ae99f8 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-09T16:31:12,196 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/.tmp/a/80cd6e76a22946538133eb5cc06f63e5 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733761871503/Put/seqid=0 2024-12-09T16:31:12,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741906_1084 (size=5523) 2024-12-09T16:31:12,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741906_1084 (size=5523) 2024-12-09T16:31:12,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741906_1084 (size=5523) 2024-12-09T16:31:12,615 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/.tmp/a/80cd6e76a22946538133eb5cc06f63e5 2024-12-09T16:31:12,648 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/.tmp/b/3b0fd4f6cf924acb957628065b3894af is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733761871498/Put/seqid=0 2024-12-09T16:31:12,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741907_1085 (size=5524) 2024-12-09T16:31:12,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741907_1085 (size=5524) 2024-12-09T16:31:12,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741907_1085 (size=5524) 2024-12-09T16:31:12,677 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/.tmp/b/3b0fd4f6cf924acb957628065b3894af 2024-12-09T16:31:12,717 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/.tmp/c/d5d8501580654d448ffa4429ed9ff124 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733761871501/Put/seqid=0 2024-12-09T16:31:12,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741908_1086 (size=5457) 2024-12-09T16:31:12,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741908_1086 (size=5457) 2024-12-09T16:31:12,743 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/.tmp/c/d5d8501580654d448ffa4429ed9ff124 2024-12-09T16:31:12,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741908_1086 (size=5457) 2024-12-09T16:31:12,752 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/.tmp/a/80cd6e76a22946538133eb5cc06f63e5 as hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/a/80cd6e76a22946538133eb5cc06f63e5 2024-12-09T16:31:12,763 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/a/80cd6e76a22946538133eb5cc06f63e5, entries=7, sequenceid=26, filesize=5.4 K 2024-12-09T16:31:12,765 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/.tmp/b/3b0fd4f6cf924acb957628065b3894af as hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/b/3b0fd4f6cf924acb957628065b3894af 2024-12-09T16:31:12,776 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/b/3b0fd4f6cf924acb957628065b3894af, entries=7, sequenceid=26, filesize=5.4 K 2024-12-09T16:31:12,778 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/.tmp/c/d5d8501580654d448ffa4429ed9ff124 as hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/c/d5d8501580654d448ffa4429ed9ff124 2024-12-09T16:31:12,789 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/c/d5d8501580654d448ffa4429ed9ff124, entries=6, sequenceid=26, filesize=5.3 K 2024-12-09T16:31:12,789 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 327eab9f241eb256cde10340b0ae99f8 in 618ms, sequenceid=26, compaction requested=false; wal=null 2024-12-09T16:31:12,791 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/recovered.edits/0000000000000000026 2024-12-09T16:31:12,793 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:12,793 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:12,795 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T16:31:12,799 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:12,802 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/hbase/data/default/testReplayEditsAfterAbortingFlush/327eab9f241eb256cde10340b0ae99f8/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-09T16:31:12,804 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 327eab9f241eb256cde10340b0ae99f8; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73198638, jitterRate=0.09074470400810242}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T16:31:12,804 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 327eab9f241eb256cde10340b0ae99f8 2024-12-09T16:31:12,805 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 327eab9f241eb256cde10340b0ae99f8: Running coprocessor pre-open hook at 1733761872142Writing region info on filesystem at 1733761872142Initializing all the Stores at 1733761872144 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761872144Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761872145 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761872145Obtaining lock to block concurrent updates at 1733761872171 (+26 ms)Preparing flush snapshotting stores in 327eab9f241eb256cde10340b0ae99f8 at 1733761872171Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733761862276.327eab9f241eb256cde10340b0ae99f8., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733761872171Flushing stores of testReplayEditsAfterAbortingFlush,,1733761862276.327eab9f241eb256cde10340b0ae99f8. at 1733761872171Flushing 327eab9f241eb256cde10340b0ae99f8/a: creating writer at 1733761872172 (+1 ms)Flushing 327eab9f241eb256cde10340b0ae99f8/a: appending metadata at 1733761872196 (+24 ms)Flushing 327eab9f241eb256cde10340b0ae99f8/a: closing flushed file at 1733761872196Flushing 327eab9f241eb256cde10340b0ae99f8/b: creating writer at 1733761872625 (+429 ms)Flushing 327eab9f241eb256cde10340b0ae99f8/b: appending metadata at 1733761872647 (+22 ms)Flushing 327eab9f241eb256cde10340b0ae99f8/b: closing flushed file at 1733761872647Flushing 327eab9f241eb256cde10340b0ae99f8/c: creating writer at 1733761872688 (+41 ms)Flushing 327eab9f241eb256cde10340b0ae99f8/c: appending metadata at 1733761872716 (+28 ms)Flushing 327eab9f241eb256cde10340b0ae99f8/c: closing flushed file at 1733761872717 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66d1d9a5: reopening flushed file at 1733761872751 (+34 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c946721: reopening flushed file at 1733761872763 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69177ed1: reopening flushed file at 1733761872777 (+14 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 327eab9f241eb256cde10340b0ae99f8 in 618ms, sequenceid=26, compaction requested=false; wal=null at 1733761872789 (+12 ms)Cleaning up temporary data from old regions at 1733761872793 (+4 ms)Running coprocessor post-open hooks at 1733761872804 (+11 ms)Region opened successfully at 1733761872805 (+1 ms) 2024-12-09T16:31:12,841 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=424 (was 420) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:58442 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-427817209-172.17.0.2-1733761835089:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-427817209-172.17.0.2-1733761835089:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:41980 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:60274 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-427817209-172.17.0.2-1733761835089:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:60272 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:58416 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:42022 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1184 (was 1124) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=592 (was 609), ProcessCount=11 (was 11), AvailableMemoryMB=8413 (was 8869) 2024-12-09T16:31:12,841 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1184 is superior to 1024 2024-12-09T16:31:12,859 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=424, OpenFileDescriptor=1184, MaxFileDescriptor=1048576, SystemLoadAverage=592, ProcessCount=11, AvailableMemoryMB=8413 2024-12-09T16:31:12,859 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1184 is superior to 1024 2024-12-09T16:31:12,882 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:31:12,885 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:31:12,886 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T16:31:12,890 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-36005291, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/hregion-36005291, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:31:12,911 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-36005291/hregion-36005291.1733761872891, exclude list is [], retry=0 2024-12-09T16:31:12,916 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:31:12,917 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:31:12,918 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:31:12,948 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-36005291/hregion-36005291.1733761872891 2024-12-09T16:31:12,953 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39661:39661),(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:37099:37099)] 2024-12-09T16:31:12,954 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 344258b370a4c42c9582881a76865008, NAME => 'testDatalossWhenInputError,,1733761872883.344258b370a4c42c9582881a76865008.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39607/hbase 2024-12-09T16:31:12,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741910_1088 (size=61) 2024-12-09T16:31:12,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741910_1088 (size=61) 2024-12-09T16:31:12,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741910_1088 (size=61) 2024-12-09T16:31:13,381 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733761872883.344258b370a4c42c9582881a76865008.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:31:13,382 INFO [StoreOpener-344258b370a4c42c9582881a76865008-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 344258b370a4c42c9582881a76865008 2024-12-09T16:31:13,384 INFO [StoreOpener-344258b370a4c42c9582881a76865008-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 344258b370a4c42c9582881a76865008 columnFamilyName a 2024-12-09T16:31:13,384 DEBUG [StoreOpener-344258b370a4c42c9582881a76865008-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:13,385 INFO [StoreOpener-344258b370a4c42c9582881a76865008-1 {}] regionserver.HStore(327): Store=344258b370a4c42c9582881a76865008/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:13,385 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 344258b370a4c42c9582881a76865008 2024-12-09T16:31:13,387 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008 2024-12-09T16:31:13,387 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008 2024-12-09T16:31:13,388 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 344258b370a4c42c9582881a76865008 2024-12-09T16:31:13,388 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 344258b370a4c42c9582881a76865008 2024-12-09T16:31:13,390 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 344258b370a4c42c9582881a76865008 2024-12-09T16:31:13,393 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T16:31:13,396 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 344258b370a4c42c9582881a76865008; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64062245, jitterRate=-0.04539816081523895}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T16:31:13,397 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 344258b370a4c42c9582881a76865008: Writing region info on filesystem at 1733761873381Initializing all the Stores at 1733761873382 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761873382Cleaning up temporary data from old regions at 1733761873388 (+6 ms)Region opened successfully at 1733761873397 (+9 ms) 2024-12-09T16:31:13,397 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 344258b370a4c42c9582881a76865008, disabling compactions & flushes 2024-12-09T16:31:13,397 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733761872883.344258b370a4c42c9582881a76865008. 2024-12-09T16:31:13,397 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733761872883.344258b370a4c42c9582881a76865008. 2024-12-09T16:31:13,397 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733761872883.344258b370a4c42c9582881a76865008. after waiting 0 ms 2024-12-09T16:31:13,398 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733761872883.344258b370a4c42c9582881a76865008. 2024-12-09T16:31:13,398 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733761872883.344258b370a4c42c9582881a76865008. 2024-12-09T16:31:13,398 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 344258b370a4c42c9582881a76865008: Waiting for close lock at 1733761873397Disabling compacts and flushes for region at 1733761873397Disabling writes for close at 1733761873397Writing region close event to WAL at 1733761873398 (+1 ms)Closed at 1733761873398 2024-12-09T16:31:13,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741909_1087 (size=93) 2024-12-09T16:31:13,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741909_1087 (size=93) 2024-12-09T16:31:13,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741909_1087 (size=93) 2024-12-09T16:31:13,420 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T16:31:13,420 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-36005291:(num 1733761872891) 2024-12-09T16:31:13,421 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T16:31:13,423 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/testdatalosswheninputerror-manual,16010,1733761872880, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:31:13,443 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733761872880/wal.1733761873424, exclude list is [], retry=0 2024-12-09T16:31:13,447 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:31:13,447 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:31:13,448 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:31:13,454 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733761872880/wal.1733761873424 2024-12-09T16:31:13,455 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:39661:39661)] 2024-12-09T16:31:13,455 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 344258b370a4c42c9582881a76865008, NAME => 'testDatalossWhenInputError,,1733761872883.344258b370a4c42c9582881a76865008.', STARTKEY => '', ENDKEY => ''} 2024-12-09T16:31:13,455 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733761872883.344258b370a4c42c9582881a76865008.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:31:13,456 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 344258b370a4c42c9582881a76865008 2024-12-09T16:31:13,456 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 344258b370a4c42c9582881a76865008 2024-12-09T16:31:13,462 INFO [StoreOpener-344258b370a4c42c9582881a76865008-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 344258b370a4c42c9582881a76865008 2024-12-09T16:31:13,463 INFO [StoreOpener-344258b370a4c42c9582881a76865008-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 344258b370a4c42c9582881a76865008 columnFamilyName a 2024-12-09T16:31:13,463 DEBUG [StoreOpener-344258b370a4c42c9582881a76865008-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:13,464 INFO [StoreOpener-344258b370a4c42c9582881a76865008-1 {}] regionserver.HStore(327): Store=344258b370a4c42c9582881a76865008/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:13,464 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 344258b370a4c42c9582881a76865008 2024-12-09T16:31:13,465 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008 2024-12-09T16:31:13,467 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008 2024-12-09T16:31:13,469 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 344258b370a4c42c9582881a76865008 2024-12-09T16:31:13,469 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 344258b370a4c42c9582881a76865008 2024-12-09T16:31:13,475 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 344258b370a4c42c9582881a76865008 2024-12-09T16:31:13,476 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 344258b370a4c42c9582881a76865008; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70738196, jitterRate=0.0540812611579895}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T16:31:13,477 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 344258b370a4c42c9582881a76865008: Writing region info on filesystem at 1733761873456Initializing all the Stores at 1733761873460 (+4 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761873460Cleaning up temporary data from old regions at 1733761873469 (+9 ms)Region opened successfully at 1733761873477 (+8 ms) 2024-12-09T16:31:13,505 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 344258b370a4c42c9582881a76865008, disabling compactions & flushes 2024-12-09T16:31:13,505 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733761872883.344258b370a4c42c9582881a76865008. 2024-12-09T16:31:13,505 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733761872883.344258b370a4c42c9582881a76865008. 2024-12-09T16:31:13,505 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733761872883.344258b370a4c42c9582881a76865008. after waiting 0 ms 2024-12-09T16:31:13,505 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733761872883.344258b370a4c42c9582881a76865008. 2024-12-09T16:31:13,509 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733761872883.344258b370a4c42c9582881a76865008. 2024-12-09T16:31:13,509 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733761872883.344258b370a4c42c9582881a76865008. 2024-12-09T16:31:13,509 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 344258b370a4c42c9582881a76865008: Waiting for close lock at 1733761873505Disabling compacts and flushes for region at 1733761873505Disabling writes for close at 1733761873505Writing region close event to WAL at 1733761873509 (+4 ms)Closed at 1733761873509 2024-12-09T16:31:13,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741911_1089 (size=838) 2024-12-09T16:31:13,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741911_1089 (size=838) 2024-12-09T16:31:13,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741911_1089 (size=838) 2024-12-09T16:31:13,549 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39607/hbase/WALs/testdatalosswheninputerror-manual,16010,1733761872880/wal.1733761873424, size=838 (838bytes) 2024-12-09T16:31:13,549 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39607/hbase/WALs/testdatalosswheninputerror-manual,16010,1733761872880/wal.1733761873424 2024-12-09T16:31:13,550 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39607/hbase/WALs/testdatalosswheninputerror-manual,16010,1733761872880/wal.1733761873424 after 1ms 2024-12-09T16:31:13,554 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/WALs/testdatalosswheninputerror-manual,16010,1733761872880/wal.1733761873424: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:31:13,555 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39607/hbase/WALs/testdatalosswheninputerror-manual,16010,1733761872880/wal.1733761873424 took 6ms 2024-12-09T16:31:13,562 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39607/hbase/WALs/testdatalosswheninputerror-manual,16010,1733761872880/wal.1733761873424 so closing down 2024-12-09T16:31:13,562 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T16:31:13,571 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733761873424.temp 2024-12-09T16:31:13,574 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008/recovered.edits/0000000000000000003-wal.1733761873424.temp 2024-12-09T16:31:13,574 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T16:31:13,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741912_1090 (size=838) 2024-12-09T16:31:13,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741912_1090 (size=838) 2024-12-09T16:31:13,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741912_1090 (size=838) 2024-12-09T16:31:13,598 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008/recovered.edits/0000000000000000003-wal.1733761873424.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-09T16:31:13,601 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008/recovered.edits/0000000000000000003-wal.1733761873424.temp to hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008/recovered.edits/0000000000000000012 2024-12-09T16:31:13,601 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:39607/hbase/WALs/testdatalosswheninputerror-manual,16010,1733761872880/wal.1733761873424, size=838, length=838, corrupted=false, cancelled=false 2024-12-09T16:31:13,601 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39607/hbase/WALs/testdatalosswheninputerror-manual,16010,1733761872880/wal.1733761873424, journal: Splitting hdfs://localhost:39607/hbase/WALs/testdatalosswheninputerror-manual,16010,1733761872880/wal.1733761873424, size=838 (838bytes) at 1733761873549Finishing writing output for hdfs://localhost:39607/hbase/WALs/testdatalosswheninputerror-manual,16010,1733761872880/wal.1733761873424 so closing down at 1733761873562 (+13 ms)Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008/recovered.edits/0000000000000000003-wal.1733761873424.temp at 1733761873574 (+12 ms)3 split writer threads finished at 1733761873574Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008/recovered.edits/0000000000000000003-wal.1733761873424.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733761873598 (+24 ms)Rename recovered edits hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008/recovered.edits/0000000000000000003-wal.1733761873424.temp to hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008/recovered.edits/0000000000000000012 at 1733761873601 (+3 ms)Processed 10 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:39607/hbase/WALs/testdatalosswheninputerror-manual,16010,1733761872880/wal.1733761873424, size=838, length=838, corrupted=false, cancelled=false at 1733761873601 2024-12-09T16:31:13,605 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39607/hbase/WALs/testdatalosswheninputerror-manual,16010,1733761872880/wal.1733761873424 to hdfs://localhost:39607/hbase/oldWALs/wal.1733761873424 2024-12-09T16:31:13,607 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008/recovered.edits/0000000000000000012 2024-12-09T16:31:13,614 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:31:14,270 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T16:31:14,283 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/testdatalosswheninputerror-manual,16010,1733761872880, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:31:14,304 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733761872880/wal.1733761874284, exclude list is [], retry=0 2024-12-09T16:31:14,318 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:31:14,319 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:31:14,319 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:31:14,344 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733761872880/wal.1733761874284 2024-12-09T16:31:14,344 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:39661:39661)] 2024-12-09T16:31:14,344 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 344258b370a4c42c9582881a76865008, NAME => 'testDatalossWhenInputError,,1733761872883.344258b370a4c42c9582881a76865008.', STARTKEY => '', ENDKEY => ''} 2024-12-09T16:31:14,345 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733761872883.344258b370a4c42c9582881a76865008.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:31:14,345 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 344258b370a4c42c9582881a76865008 2024-12-09T16:31:14,345 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 344258b370a4c42c9582881a76865008 2024-12-09T16:31:14,386 INFO [StoreOpener-344258b370a4c42c9582881a76865008-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 344258b370a4c42c9582881a76865008 2024-12-09T16:31:14,390 INFO [StoreOpener-344258b370a4c42c9582881a76865008-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 344258b370a4c42c9582881a76865008 columnFamilyName a 2024-12-09T16:31:14,390 DEBUG [StoreOpener-344258b370a4c42c9582881a76865008-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:14,392 INFO [StoreOpener-344258b370a4c42c9582881a76865008-1 {}] regionserver.HStore(327): Store=344258b370a4c42c9582881a76865008/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:14,394 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 344258b370a4c42c9582881a76865008 2024-12-09T16:31:14,400 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008 2024-12-09T16:31:14,417 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008 2024-12-09T16:31:14,426 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008/recovered.edits/0000000000000000012 2024-12-09T16:31:14,435 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:31:14,437 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008/recovered.edits/0000000000000000012 2024-12-09T16:31:14,437 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 344258b370a4c42c9582881a76865008 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-09T16:31:14,475 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008/.tmp/a/dfffb6495789470e988d6cfe44bc05d9 is 79, key is testDatalossWhenInputError/a:x0/1733761873477/Put/seqid=0 2024-12-09T16:31:14,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741914_1092 (size=5808) 2024-12-09T16:31:14,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741914_1092 (size=5808) 2024-12-09T16:31:14,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741914_1092 (size=5808) 2024-12-09T16:31:14,538 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008/.tmp/a/dfffb6495789470e988d6cfe44bc05d9 2024-12-09T16:31:14,571 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008/.tmp/a/dfffb6495789470e988d6cfe44bc05d9 as hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008/a/dfffb6495789470e988d6cfe44bc05d9 2024-12-09T16:31:14,606 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008/a/dfffb6495789470e988d6cfe44bc05d9, entries=10, sequenceid=12, filesize=5.7 K 2024-12-09T16:31:14,607 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 344258b370a4c42c9582881a76865008 in 169ms, sequenceid=12, compaction requested=false; wal=null 2024-12-09T16:31:14,608 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008/recovered.edits/0000000000000000012 2024-12-09T16:31:14,614 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 344258b370a4c42c9582881a76865008 2024-12-09T16:31:14,614 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 344258b370a4c42c9582881a76865008 2024-12-09T16:31:14,623 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 344258b370a4c42c9582881a76865008 2024-12-09T16:31:14,642 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-09T16:31:14,650 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 344258b370a4c42c9582881a76865008; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60891806, jitterRate=-0.09264138340950012}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T16:31:14,651 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 344258b370a4c42c9582881a76865008: Writing region info on filesystem at 1733761874345Initializing all the Stores at 1733761874364 (+19 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761874364Obtaining lock to block concurrent updates at 1733761874437 (+73 ms)Preparing flush snapshotting stores in 344258b370a4c42c9582881a76865008 at 1733761874437Finished memstore snapshotting testDatalossWhenInputError,,1733761872883.344258b370a4c42c9582881a76865008., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733761874437Flushing stores of testDatalossWhenInputError,,1733761872883.344258b370a4c42c9582881a76865008. at 1733761874437Flushing 344258b370a4c42c9582881a76865008/a: creating writer at 1733761874437Flushing 344258b370a4c42c9582881a76865008/a: appending metadata at 1733761874474 (+37 ms)Flushing 344258b370a4c42c9582881a76865008/a: closing flushed file at 1733761874474Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16b5b554: reopening flushed file at 1733761874567 (+93 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 344258b370a4c42c9582881a76865008 in 169ms, sequenceid=12, compaction requested=false; wal=null at 1733761874607 (+40 ms)Cleaning up temporary data from old regions at 1733761874614 (+7 ms)Region opened successfully at 1733761874651 (+37 ms) 2024-12-09T16:31:14,662 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 344258b370a4c42c9582881a76865008, NAME => 'testDatalossWhenInputError,,1733761872883.344258b370a4c42c9582881a76865008.', STARTKEY => '', ENDKEY => ''} 2024-12-09T16:31:14,662 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733761872883.344258b370a4c42c9582881a76865008.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:31:14,662 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 344258b370a4c42c9582881a76865008 2024-12-09T16:31:14,662 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 344258b370a4c42c9582881a76865008 2024-12-09T16:31:14,669 INFO [StoreOpener-344258b370a4c42c9582881a76865008-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 344258b370a4c42c9582881a76865008 2024-12-09T16:31:14,671 INFO [StoreOpener-344258b370a4c42c9582881a76865008-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 344258b370a4c42c9582881a76865008 columnFamilyName a 2024-12-09T16:31:14,671 DEBUG [StoreOpener-344258b370a4c42c9582881a76865008-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:14,690 DEBUG [StoreOpener-344258b370a4c42c9582881a76865008-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008/a/dfffb6495789470e988d6cfe44bc05d9 2024-12-09T16:31:14,690 INFO [StoreOpener-344258b370a4c42c9582881a76865008-1 {}] regionserver.HStore(327): Store=344258b370a4c42c9582881a76865008/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:14,690 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 344258b370a4c42c9582881a76865008 2024-12-09T16:31:14,691 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008 2024-12-09T16:31:14,703 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008 2024-12-09T16:31:14,708 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 344258b370a4c42c9582881a76865008 2024-12-09T16:31:14,708 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 344258b370a4c42c9582881a76865008 2024-12-09T16:31:14,720 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 344258b370a4c42c9582881a76865008 2024-12-09T16:31:14,738 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/hbase/data/default/testDatalossWhenInputError/344258b370a4c42c9582881a76865008/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-09T16:31:14,740 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 344258b370a4c42c9582881a76865008; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67881167, jitterRate=0.011508211493492126}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T16:31:14,741 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 344258b370a4c42c9582881a76865008: Writing region info on filesystem at 1733761874662Initializing all the Stores at 1733761874668 (+6 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761874669 (+1 ms)Cleaning up temporary data from old regions at 1733761874708 (+39 ms)Region opened successfully at 1733761874741 (+33 ms) 2024-12-09T16:31:14,767 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=434 (was 424) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-427817209-172.17.0.2-1733761835089:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:42080 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:58530 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-427817209-172.17.0.2-1733761835089:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:60274 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:60368 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:58416 [Waiting for operation #26] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:42022 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-427817209-172.17.0.2-1733761835089:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1268 (was 1184) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=592 (was 592), ProcessCount=11 (was 11), AvailableMemoryMB=8054 (was 8413) 2024-12-09T16:31:14,767 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1268 is superior to 1024 2024-12-09T16:31:14,798 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=434, OpenFileDescriptor=1268, MaxFileDescriptor=1048576, SystemLoadAverage=592, ProcessCount=11, AvailableMemoryMB=8051 2024-12-09T16:31:14,799 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1268 is superior to 1024 2024-12-09T16:31:14,823 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:31:14,826 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:31:14,827 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T16:31:14,835 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-63810851, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/hregion-63810851, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:31:14,854 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-63810851/hregion-63810851.1733761874835, exclude list is [], retry=0 2024-12-09T16:31:14,858 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:31:14,865 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:31:14,895 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:31:14,900 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-63810851/hregion-63810851.1733761874835 2024-12-09T16:31:14,902 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:39661:39661),(127.0.0.1/127.0.0.1:37099:37099)] 2024-12-09T16:31:14,902 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8f00cb31d0626567f366f081b6701d34, NAME => 'testCompactedBulkLoadedFiles,,1733761874824.8f00cb31d0626567f366f081b6701d34.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39607/hbase 2024-12-09T16:31:14,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741916_1094 (size=63) 2024-12-09T16:31:14,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741916_1094 (size=63) 2024-12-09T16:31:14,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741916_1094 (size=63) 2024-12-09T16:31:14,936 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733761874824.8f00cb31d0626567f366f081b6701d34.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:31:14,942 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:14,948 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f00cb31d0626567f366f081b6701d34 columnFamilyName a 2024-12-09T16:31:14,948 DEBUG [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:14,949 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.HStore(327): Store=8f00cb31d0626567f366f081b6701d34/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:14,949 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:14,951 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f00cb31d0626567f366f081b6701d34 columnFamilyName b 2024-12-09T16:31:14,951 DEBUG [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:14,955 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.HStore(327): Store=8f00cb31d0626567f366f081b6701d34/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:14,955 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:14,961 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f00cb31d0626567f366f081b6701d34 columnFamilyName c 2024-12-09T16:31:14,961 DEBUG [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:14,964 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.HStore(327): Store=8f00cb31d0626567f366f081b6701d34/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:14,965 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:14,966 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:14,967 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:14,968 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:14,968 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:14,969 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T16:31:14,970 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:14,977 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T16:31:14,982 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8f00cb31d0626567f366f081b6701d34; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69167658, jitterRate=0.03067842125892639}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T16:31:14,983 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8f00cb31d0626567f366f081b6701d34: Writing region info on filesystem at 1733761874936Initializing all the Stores at 1733761874939 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761874939Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761874942 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761874942Cleaning up temporary data from old regions at 1733761874968 (+26 ms)Region opened successfully at 1733761874983 (+15 ms) 2024-12-09T16:31:14,983 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8f00cb31d0626567f366f081b6701d34, disabling compactions & flushes 2024-12-09T16:31:14,983 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733761874824.8f00cb31d0626567f366f081b6701d34. 2024-12-09T16:31:14,983 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733761874824.8f00cb31d0626567f366f081b6701d34. 2024-12-09T16:31:14,983 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733761874824.8f00cb31d0626567f366f081b6701d34. after waiting 0 ms 2024-12-09T16:31:14,983 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733761874824.8f00cb31d0626567f366f081b6701d34. 2024-12-09T16:31:14,988 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733761874824.8f00cb31d0626567f366f081b6701d34. 2024-12-09T16:31:14,988 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8f00cb31d0626567f366f081b6701d34: Waiting for close lock at 1733761874983Disabling compacts and flushes for region at 1733761874983Disabling writes for close at 1733761874983Writing region close event to WAL at 1733761874988 (+5 ms)Closed at 1733761874988 2024-12-09T16:31:14,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741915_1093 (size=93) 2024-12-09T16:31:15,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741915_1093 (size=93) 2024-12-09T16:31:15,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741915_1093 (size=93) 2024-12-09T16:31:15,014 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T16:31:15,014 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-63810851:(num 1733761874835) 2024-12-09T16:31:15,014 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T16:31:15,019 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733761874822, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:31:15,040 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733761874822/wal.1733761875019, exclude list is [], retry=0 2024-12-09T16:31:15,046 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:31:15,047 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:31:15,047 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:31:15,079 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733761874822/wal.1733761875019 2024-12-09T16:31:15,082 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39661:39661),(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:37099:37099)] 2024-12-09T16:31:15,082 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f00cb31d0626567f366f081b6701d34, NAME => 'testCompactedBulkLoadedFiles,,1733761874824.8f00cb31d0626567f366f081b6701d34.', STARTKEY => '', ENDKEY => ''} 2024-12-09T16:31:15,083 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733761874824.8f00cb31d0626567f366f081b6701d34.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:31:15,083 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:15,083 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:15,092 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:15,096 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f00cb31d0626567f366f081b6701d34 columnFamilyName a 2024-12-09T16:31:15,096 DEBUG [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:15,103 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.HStore(327): Store=8f00cb31d0626567f366f081b6701d34/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:15,103 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:15,107 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f00cb31d0626567f366f081b6701d34 columnFamilyName b 2024-12-09T16:31:15,107 DEBUG [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:15,114 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.HStore(327): Store=8f00cb31d0626567f366f081b6701d34/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:15,114 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:15,117 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f00cb31d0626567f366f081b6701d34 columnFamilyName c 2024-12-09T16:31:15,117 DEBUG [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:15,126 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.HStore(327): Store=8f00cb31d0626567f366f081b6701d34/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:15,128 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:15,136 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:15,139 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:15,143 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:15,143 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:15,144 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T16:31:15,151 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:15,153 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8f00cb31d0626567f366f081b6701d34; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59031750, jitterRate=-0.12035837769508362}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T16:31:15,153 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8f00cb31d0626567f366f081b6701d34: Writing region info on filesystem at 1733761875083Initializing all the Stores at 1733761875090 (+7 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761875090Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761875092 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761875092Cleaning up temporary data from old regions at 1733761875143 (+51 ms)Region opened successfully at 1733761875153 (+10 ms) 2024-12-09T16:31:15,162 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733761875162/Put/seqid=0 2024-12-09T16:31:15,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741918_1096 (size=4875) 2024-12-09T16:31:15,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741918_1096 (size=4875) 2024-12-09T16:31:15,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741918_1096 (size=4875) 2024-12-09T16:31:15,584 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733761875584/Put/seqid=0 2024-12-09T16:31:15,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741919_1097 (size=4875) 2024-12-09T16:31:15,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741919_1097 (size=4875) 2024-12-09T16:31:15,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741919_1097 (size=4875) 2024-12-09T16:31:16,026 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733761876025/Put/seqid=0 2024-12-09T16:31:16,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741920_1098 (size=4875) 2024-12-09T16:31:16,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741920_1098 (size=4875) 2024-12-09T16:31:16,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741920_1098 (size=4875) 2024-12-09T16:31:16,044 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:39607/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 8f00cb31d0626567f366f081b6701d34/a 2024-12-09T16:31:16,051 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-09T16:31:16,051 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T16:31:16,051 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:39607/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 8f00cb31d0626567f366f081b6701d34/a 2024-12-09T16:31:16,064 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-09T16:31:16,064 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T16:31:16,064 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:39607/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 8f00cb31d0626567f366f081b6701d34/a 2024-12-09T16:31:16,070 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-09T16:31:16,070 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T16:31:16,070 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8f00cb31d0626567f366f081b6701d34 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-09T16:31:16,090 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/.tmp/a/c66ad3b3b5bd4947bc03f9fd5d01b057 is 55, key is testCompactedBulkLoadedFiles/a:a/1733761875155/Put/seqid=0 2024-12-09T16:31:16,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741921_1099 (size=5107) 2024-12-09T16:31:16,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741921_1099 (size=5107) 2024-12-09T16:31:16,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741921_1099 (size=5107) 2024-12-09T16:31:16,510 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/.tmp/a/c66ad3b3b5bd4947bc03f9fd5d01b057 2024-12-09T16:31:16,529 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/.tmp/a/c66ad3b3b5bd4947bc03f9fd5d01b057 as hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/c66ad3b3b5bd4947bc03f9fd5d01b057 2024-12-09T16:31:16,546 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/c66ad3b3b5bd4947bc03f9fd5d01b057, entries=1, sequenceid=4, filesize=5.0 K 2024-12-09T16:31:16,554 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 8f00cb31d0626567f366f081b6701d34 in 484ms, sequenceid=4, compaction requested=false 2024-12-09T16:31:16,554 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8f00cb31d0626567f366f081b6701d34: 2024-12-09T16:31:16,558 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/63ffb8cc2dae44f79b4709d19cf1acf5_SeqId_4_ 2024-12-09T16:31:16,559 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/f56206ffb9fb4382846d0b43152aabed_SeqId_4_ 2024-12-09T16:31:16,561 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/0fe42de7699547ee9c7f246a198bf878_SeqId_4_ 2024-12-09T16:31:16,562 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:39607/hbase/testCompactedBulkLoadedFiles/hfile0 into 8f00cb31d0626567f366f081b6701d34/a as hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/63ffb8cc2dae44f79b4709d19cf1acf5_SeqId_4_ - updating store file list. 2024-12-09T16:31:16,574 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 63ffb8cc2dae44f79b4709d19cf1acf5_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T16:31:16,574 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/63ffb8cc2dae44f79b4709d19cf1acf5_SeqId_4_ into 8f00cb31d0626567f366f081b6701d34/a 2024-12-09T16:31:16,574 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:39607/hbase/testCompactedBulkLoadedFiles/hfile0 into 8f00cb31d0626567f366f081b6701d34/a (new location: hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/63ffb8cc2dae44f79b4709d19cf1acf5_SeqId_4_) 2024-12-09T16:31:16,576 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:39607/hbase/testCompactedBulkLoadedFiles/hfile1 into 8f00cb31d0626567f366f081b6701d34/a as hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/f56206ffb9fb4382846d0b43152aabed_SeqId_4_ - updating store file list. 2024-12-09T16:31:16,583 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for f56206ffb9fb4382846d0b43152aabed_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T16:31:16,583 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/f56206ffb9fb4382846d0b43152aabed_SeqId_4_ into 8f00cb31d0626567f366f081b6701d34/a 2024-12-09T16:31:16,583 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:39607/hbase/testCompactedBulkLoadedFiles/hfile1 into 8f00cb31d0626567f366f081b6701d34/a (new location: hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/f56206ffb9fb4382846d0b43152aabed_SeqId_4_) 2024-12-09T16:31:16,585 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:39607/hbase/testCompactedBulkLoadedFiles/hfile2 into 8f00cb31d0626567f366f081b6701d34/a as hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/0fe42de7699547ee9c7f246a198bf878_SeqId_4_ - updating store file list. 2024-12-09T16:31:16,592 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 0fe42de7699547ee9c7f246a198bf878_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T16:31:16,592 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/0fe42de7699547ee9c7f246a198bf878_SeqId_4_ into 8f00cb31d0626567f366f081b6701d34/a 2024-12-09T16:31:16,592 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:39607/hbase/testCompactedBulkLoadedFiles/hfile2 into 8f00cb31d0626567f366f081b6701d34/a (new location: hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/0fe42de7699547ee9c7f246a198bf878_SeqId_4_) 2024-12-09T16:31:16,611 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T16:31:16,612 DEBUG [Time-limited test {}] regionserver.HStore(1541): 8f00cb31d0626567f366f081b6701d34/a is initiating major compaction (all files) 2024-12-09T16:31:16,612 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 8f00cb31d0626567f366f081b6701d34/a in testCompactedBulkLoadedFiles,,1733761874824.8f00cb31d0626567f366f081b6701d34. 2024-12-09T16:31:16,612 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/c66ad3b3b5bd4947bc03f9fd5d01b057, hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/63ffb8cc2dae44f79b4709d19cf1acf5_SeqId_4_, hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/f56206ffb9fb4382846d0b43152aabed_SeqId_4_, hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/0fe42de7699547ee9c7f246a198bf878_SeqId_4_] into tmpdir=hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/.tmp, totalSize=19.3 K 2024-12-09T16:31:16,613 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c66ad3b3b5bd4947bc03f9fd5d01b057, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733761875155 2024-12-09T16:31:16,614 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 63ffb8cc2dae44f79b4709d19cf1acf5_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-09T16:31:16,614 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting f56206ffb9fb4382846d0b43152aabed_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-09T16:31:16,617 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 0fe42de7699547ee9c7f246a198bf878_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-09T16:31:16,653 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/.tmp/a/e696cdecd2d64c0fbdcc55b291fa3464 is 55, key is testCompactedBulkLoadedFiles/a:a/1733761875155/Put/seqid=0 2024-12-09T16:31:16,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741922_1100 (size=6154) 2024-12-09T16:31:16,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741922_1100 (size=6154) 2024-12-09T16:31:16,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741922_1100 (size=6154) 2024-12-09T16:31:16,696 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/.tmp/a/e696cdecd2d64c0fbdcc55b291fa3464 as hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/e696cdecd2d64c0fbdcc55b291fa3464 2024-12-09T16:31:16,721 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 8f00cb31d0626567f366f081b6701d34/a of 8f00cb31d0626567f366f081b6701d34 into e696cdecd2d64c0fbdcc55b291fa3464(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T16:31:16,721 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 8f00cb31d0626567f366f081b6701d34: 2024-12-09T16:31:16,721 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-09T16:31:16,721 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-09T16:31:16,773 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39607/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733761874822/wal.1733761875019, size=0 (0bytes) 2024-12-09T16:31:16,773 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:39607/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733761874822/wal.1733761875019 might be still open, length is 0 2024-12-09T16:31:16,773 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39607/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733761874822/wal.1733761875019 2024-12-09T16:31:16,774 WARN [IPC Server handler 0 on default port 39607 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733761874822/wal.1733761875019 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-12-09T16:31:16,774 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39607/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733761874822/wal.1733761875019 after 1ms 2024-12-09T16:31:17,075 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T16:31:17,924 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:60530 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:39721:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60530 dst: /127.0.0.1:39721 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39721 remote=/127.0.0.1:60530]. Total timeout mills is 60000, 58795 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:31:17,926 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:51252 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:36895:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51252 dst: /127.0.0.1:36895 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:31:17,927 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:51308 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:41451:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51308 dst: /127.0.0.1:41451 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:31:17,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741917_1101 (size=1368) 2024-12-09T16:31:17,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741917_1101 (size=1368) 2024-12-09T16:31:17,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741917_1101 (size=1368) 2024-12-09T16:31:20,234 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-09T16:31:20,234 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-09T16:31:20,235 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-09T16:31:20,235 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-09T16:31:20,775 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:39607/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733761874822/wal.1733761875019 after 4002ms 2024-12-09T16:31:20,779 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733761874822/wal.1733761875019: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:31:20,779 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39607/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733761874822/wal.1733761875019 took 4006ms 2024-12-09T16:31:20,784 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:39607/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733761874822/wal.1733761875019; continuing. 2024-12-09T16:31:20,784 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39607/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733761874822/wal.1733761875019 so closing down 2024-12-09T16:31:20,784 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T16:31:20,788 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733761875019.temp 2024-12-09T16:31:20,790 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/recovered.edits/0000000000000000003-wal.1733761875019.temp 2024-12-09T16:31:20,790 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T16:31:20,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741923_1102 (size=635) 2024-12-09T16:31:20,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741923_1102 (size=635) 2024-12-09T16:31:20,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741923_1102 (size=635) 2024-12-09T16:31:20,815 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/recovered.edits/0000000000000000003-wal.1733761875019.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-09T16:31:20,817 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/recovered.edits/0000000000000000003-wal.1733761875019.temp to hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/recovered.edits/0000000000000000008 2024-12-09T16:31:20,817 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 35 ms; skipped=3; WAL=hdfs://localhost:39607/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733761874822/wal.1733761875019, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T16:31:20,818 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39607/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733761874822/wal.1733761875019, journal: Splitting hdfs://localhost:39607/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733761874822/wal.1733761875019, size=0 (0bytes) at 1733761876773Finishing writing output for hdfs://localhost:39607/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733761874822/wal.1733761875019 so closing down at 1733761880784 (+4011 ms)Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/recovered.edits/0000000000000000003-wal.1733761875019.temp at 1733761880790 (+6 ms)3 split writer threads finished at 1733761880790Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/recovered.edits/0000000000000000003-wal.1733761875019.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733761880816 (+26 ms)Rename recovered edits hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/recovered.edits/0000000000000000003-wal.1733761875019.temp to hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/recovered.edits/0000000000000000008 at 1733761880817 (+1 ms)Processed 5 edits across 1 Regions in 35 ms; skipped=3; WAL=hdfs://localhost:39607/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733761874822/wal.1733761875019, size=0, length=0, corrupted=false, cancelled=false at 1733761880817 2024-12-09T16:31:20,820 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39607/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733761874822/wal.1733761875019 to hdfs://localhost:39607/hbase/oldWALs/wal.1733761875019 2024-12-09T16:31:20,823 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/recovered.edits/0000000000000000008 2024-12-09T16:31:20,824 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T16:31:20,827 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733761874822, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:31:20,847 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733761874822/wal.1733761880827, exclude list is [], retry=0 2024-12-09T16:31:20,850 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:31:20,851 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:31:20,851 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:31:20,856 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733761874822/wal.1733761880827 2024-12-09T16:31:20,858 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39661:39661),(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:35519:35519)] 2024-12-09T16:31:20,858 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f00cb31d0626567f366f081b6701d34, NAME => 'testCompactedBulkLoadedFiles,,1733761874824.8f00cb31d0626567f366f081b6701d34.', STARTKEY => '', ENDKEY => ''} 2024-12-09T16:31:20,859 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733761874824.8f00cb31d0626567f366f081b6701d34.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:31:20,859 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:20,859 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:20,861 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:20,862 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f00cb31d0626567f366f081b6701d34 columnFamilyName a 2024-12-09T16:31:20,862 DEBUG [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:20,874 DEBUG [StoreFileOpener-8f00cb31d0626567f366f081b6701d34-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 0fe42de7699547ee9c7f246a198bf878_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T16:31:20,874 DEBUG [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/0fe42de7699547ee9c7f246a198bf878_SeqId_4_ 2024-12-09T16:31:20,885 DEBUG [StoreFileOpener-8f00cb31d0626567f366f081b6701d34-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 63ffb8cc2dae44f79b4709d19cf1acf5_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T16:31:20,885 DEBUG [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/63ffb8cc2dae44f79b4709d19cf1acf5_SeqId_4_ 2024-12-09T16:31:20,895 DEBUG [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/c66ad3b3b5bd4947bc03f9fd5d01b057 2024-12-09T16:31:20,903 DEBUG [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/e696cdecd2d64c0fbdcc55b291fa3464 2024-12-09T16:31:20,913 DEBUG [StoreFileOpener-8f00cb31d0626567f366f081b6701d34-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for f56206ffb9fb4382846d0b43152aabed_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T16:31:20,913 DEBUG [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/f56206ffb9fb4382846d0b43152aabed_SeqId_4_ 2024-12-09T16:31:20,914 WARN [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/0fe42de7699547ee9c7f246a198bf878_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@46db326c 2024-12-09T16:31:20,914 WARN [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/63ffb8cc2dae44f79b4709d19cf1acf5_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@46db326c 2024-12-09T16:31:20,914 WARN [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/c66ad3b3b5bd4947bc03f9fd5d01b057 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@46db326c 2024-12-09T16:31:20,914 WARN [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/f56206ffb9fb4382846d0b43152aabed_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@46db326c 2024-12-09T16:31:20,914 DEBUG [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/0fe42de7699547ee9c7f246a198bf878_SeqId_4_, hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/63ffb8cc2dae44f79b4709d19cf1acf5_SeqId_4_, hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/c66ad3b3b5bd4947bc03f9fd5d01b057, hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/f56206ffb9fb4382846d0b43152aabed_SeqId_4_] to archive 2024-12-09T16:31:20,916 DEBUG [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T16:31:20,924 DEBUG [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/0fe42de7699547ee9c7f246a198bf878_SeqId_4_ to hdfs://localhost:39607/hbase/archive/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/0fe42de7699547ee9c7f246a198bf878_SeqId_4_ 2024-12-09T16:31:20,926 DEBUG [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/63ffb8cc2dae44f79b4709d19cf1acf5_SeqId_4_ to hdfs://localhost:39607/hbase/archive/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/63ffb8cc2dae44f79b4709d19cf1acf5_SeqId_4_ 2024-12-09T16:31:20,929 DEBUG [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/c66ad3b3b5bd4947bc03f9fd5d01b057 to hdfs://localhost:39607/hbase/archive/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/c66ad3b3b5bd4947bc03f9fd5d01b057 2024-12-09T16:31:20,931 DEBUG [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/f56206ffb9fb4382846d0b43152aabed_SeqId_4_ to hdfs://localhost:39607/hbase/archive/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/a/f56206ffb9fb4382846d0b43152aabed_SeqId_4_ 2024-12-09T16:31:20,931 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.HStore(327): Store=8f00cb31d0626567f366f081b6701d34/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:20,931 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:20,932 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f00cb31d0626567f366f081b6701d34 columnFamilyName b 2024-12-09T16:31:20,932 DEBUG [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:20,933 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.HStore(327): Store=8f00cb31d0626567f366f081b6701d34/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:20,933 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:20,934 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f00cb31d0626567f366f081b6701d34 columnFamilyName c 2024-12-09T16:31:20,934 DEBUG [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:20,935 INFO [StoreOpener-8f00cb31d0626567f366f081b6701d34-1 {}] regionserver.HStore(327): Store=8f00cb31d0626567f366f081b6701d34/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:20,935 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:20,937 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:20,939 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:20,939 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/recovered.edits/0000000000000000008 2024-12-09T16:31:20,943 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:31:20,948 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 8f00cb31d0626567f366f081b6701d34 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "8f00cb31d0626567f366f081b6701d34" family_name: "a" compaction_input: "c66ad3b3b5bd4947bc03f9fd5d01b057" compaction_input: "63ffb8cc2dae44f79b4709d19cf1acf5_SeqId_4_" compaction_input: "f56206ffb9fb4382846d0b43152aabed_SeqId_4_" compaction_input: "0fe42de7699547ee9c7f246a198bf878_SeqId_4_" compaction_output: "e696cdecd2d64c0fbdcc55b291fa3464" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733761874824.8f00cb31d0626567f366f081b6701d34." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-09T16:31:20,948 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-09T16:31:20,949 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/recovered.edits/0000000000000000008 2024-12-09T16:31:20,950 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/recovered.edits/0000000000000000008 2024-12-09T16:31:20,952 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:20,952 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:20,954 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T16:31:20,956 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8f00cb31d0626567f366f081b6701d34 2024-12-09T16:31:20,962 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/hbase/data/default/testCompactedBulkLoadedFiles/8f00cb31d0626567f366f081b6701d34/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-09T16:31:20,965 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8f00cb31d0626567f366f081b6701d34; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64547433, jitterRate=-0.038168296217918396}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T16:31:20,966 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8f00cb31d0626567f366f081b6701d34: Writing region info on filesystem at 1733761880859Initializing all the Stores at 1733761880860 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761880860Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761880861 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761880861Cleaning up temporary data from old regions at 1733761880952 (+91 ms)Region opened successfully at 1733761880965 (+13 ms) 2024-12-09T16:31:20,970 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8f00cb31d0626567f366f081b6701d34, disabling compactions & flushes 2024-12-09T16:31:20,970 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733761874824.8f00cb31d0626567f366f081b6701d34. 2024-12-09T16:31:20,970 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733761874824.8f00cb31d0626567f366f081b6701d34. 2024-12-09T16:31:20,970 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733761874824.8f00cb31d0626567f366f081b6701d34. after waiting 0 ms 2024-12-09T16:31:20,970 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733761874824.8f00cb31d0626567f366f081b6701d34. 2024-12-09T16:31:20,971 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733761874824.8f00cb31d0626567f366f081b6701d34. 2024-12-09T16:31:20,971 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8f00cb31d0626567f366f081b6701d34: Waiting for close lock at 1733761880970Disabling compacts and flushes for region at 1733761880970Disabling writes for close at 1733761880970Writing region close event to WAL at 1733761880971 (+1 ms)Closed at 1733761880971 2024-12-09T16:31:20,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741924_1103 (size=93) 2024-12-09T16:31:20,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741924_1103 (size=93) 2024-12-09T16:31:20,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741924_1103 (size=93) 2024-12-09T16:31:20,983 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T16:31:20,984 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733761880827) 2024-12-09T16:31:21,003 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=443 (was 434) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1985134411_22 at /127.0.0.1:51362 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:39607 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1985134411_22 at /127.0.0.1:60610 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:39607 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1985134411_22 at /127.0.0.1:51316 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1268) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=601 (was 592) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7288 (was 8051) 2024-12-09T16:31:21,004 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-12-09T16:31:21,027 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=443, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=601, ProcessCount=11, AvailableMemoryMB=7287 2024-12-09T16:31:21,027 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-12-09T16:31:21,049 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:31:21,052 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T16:31:21,053 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T16:31:21,058 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-43837901, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/hregion-43837901, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:31:21,079 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-43837901/hregion-43837901.1733761881059, exclude list is [], retry=0 2024-12-09T16:31:21,084 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:31:21,085 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:31:21,091 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:31:21,097 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-43837901/hregion-43837901.1733761881059 2024-12-09T16:31:21,101 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:39661:39661),(127.0.0.1/127.0.0.1:35519:35519)] 2024-12-09T16:31:21,101 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d14c45326746921f20d52909371268eb, NAME => 'testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39607/hbase 2024-12-09T16:31:21,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741926_1105 (size=67) 2024-12-09T16:31:21,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741926_1105 (size=67) 2024-12-09T16:31:21,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741926_1105 (size=67) 2024-12-09T16:31:21,132 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:31:21,134 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,137 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d14c45326746921f20d52909371268eb columnFamilyName a 2024-12-09T16:31:21,137 DEBUG [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:21,138 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.HStore(327): Store=d14c45326746921f20d52909371268eb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:21,138 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,140 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d14c45326746921f20d52909371268eb columnFamilyName b 2024-12-09T16:31:21,141 DEBUG [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:21,141 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.HStore(327): Store=d14c45326746921f20d52909371268eb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:21,142 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,144 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d14c45326746921f20d52909371268eb columnFamilyName c 2024-12-09T16:31:21,144 DEBUG [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:21,145 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.HStore(327): Store=d14c45326746921f20d52909371268eb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:21,146 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,147 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,147 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,149 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,149 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,152 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T16:31:21,154 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,159 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T16:31:21,160 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d14c45326746921f20d52909371268eb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65359697, jitterRate=-0.026064619421958923}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T16:31:21,160 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d14c45326746921f20d52909371268eb: Writing region info on filesystem at 1733761881133Initializing all the Stores at 1733761881134 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761881134Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761881134Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761881134Cleaning up temporary data from old regions at 1733761881149 (+15 ms)Region opened successfully at 1733761881160 (+11 ms) 2024-12-09T16:31:21,160 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d14c45326746921f20d52909371268eb, disabling compactions & flushes 2024-12-09T16:31:21,160 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb. 2024-12-09T16:31:21,160 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb. 2024-12-09T16:31:21,160 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb. after waiting 0 ms 2024-12-09T16:31:21,160 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb. 2024-12-09T16:31:21,162 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb. 2024-12-09T16:31:21,162 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d14c45326746921f20d52909371268eb: Waiting for close lock at 1733761881160Disabling compacts and flushes for region at 1733761881160Disabling writes for close at 1733761881160Writing region close event to WAL at 1733761881162 (+2 ms)Closed at 1733761881162 2024-12-09T16:31:21,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741925_1104 (size=93) 2024-12-09T16:31:21,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741925_1104 (size=93) 2024-12-09T16:31:21,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741925_1104 (size=93) 2024-12-09T16:31:21,177 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T16:31:21,177 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-43837901:(num 1733761881059) 2024-12-09T16:31:21,177 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T16:31:21,186 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:31:21,204 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881186, exclude list is [], retry=0 2024-12-09T16:31:21,209 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:31:21,210 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:31:21,211 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:31:21,223 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881186 2024-12-09T16:31:21,226 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39661:39661),(127.0.0.1/127.0.0.1:35519:35519),(127.0.0.1/127.0.0.1:37099:37099)] 2024-12-09T16:31:21,226 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d14c45326746921f20d52909371268eb, NAME => 'testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb.', STARTKEY => '', ENDKEY => ''} 2024-12-09T16:31:21,226 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:31:21,227 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,227 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,231 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,239 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d14c45326746921f20d52909371268eb columnFamilyName a 2024-12-09T16:31:21,239 DEBUG [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:21,240 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.HStore(327): Store=d14c45326746921f20d52909371268eb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:21,240 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,242 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d14c45326746921f20d52909371268eb columnFamilyName b 2024-12-09T16:31:21,242 DEBUG [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:21,243 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.HStore(327): Store=d14c45326746921f20d52909371268eb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:21,243 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,244 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d14c45326746921f20d52909371268eb columnFamilyName c 2024-12-09T16:31:21,245 DEBUG [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:21,245 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.HStore(327): Store=d14c45326746921f20d52909371268eb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:21,246 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,247 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,249 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,250 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,251 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,251 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T16:31:21,254 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,256 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d14c45326746921f20d52909371268eb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75373142, jitterRate=0.12314733862876892}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T16:31:21,256 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d14c45326746921f20d52909371268eb: Writing region info on filesystem at 1733761881227Initializing all the Stores at 1733761881230 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761881230Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761881231 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761881231Cleaning up temporary data from old regions at 1733761881251 (+20 ms)Region opened successfully at 1733761881256 (+5 ms) 2024-12-09T16:31:21,287 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d14c45326746921f20d52909371268eb 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-09T16:31:21,318 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/.tmp/a/4ca437f65bd14feba5b7989d11c9516c is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733761881256/Put/seqid=0 2024-12-09T16:31:21,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741928_1107 (size=5958) 2024-12-09T16:31:21,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741928_1107 (size=5958) 2024-12-09T16:31:21,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741928_1107 (size=5958) 2024-12-09T16:31:21,335 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/.tmp/a/4ca437f65bd14feba5b7989d11c9516c 2024-12-09T16:31:21,349 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/.tmp/a/4ca437f65bd14feba5b7989d11c9516c as hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/a/4ca437f65bd14feba5b7989d11c9516c 2024-12-09T16:31:21,371 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/a/4ca437f65bd14feba5b7989d11c9516c, entries=10, sequenceid=13, filesize=5.8 K 2024-12-09T16:31:21,374 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for d14c45326746921f20d52909371268eb in 87ms, sequenceid=13, compaction requested=false 2024-12-09T16:31:21,374 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d14c45326746921f20d52909371268eb: 2024-12-09T16:31:21,422 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d14c45326746921f20d52909371268eb, disabling compactions & flushes 2024-12-09T16:31:21,422 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb. 2024-12-09T16:31:21,422 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb. 2024-12-09T16:31:21,422 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb. after waiting 0 ms 2024-12-09T16:31:21,422 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb. 2024-12-09T16:31:21,443 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb. 2024-12-09T16:31:21,443 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb. 2024-12-09T16:31:21,443 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d14c45326746921f20d52909371268eb: Waiting for close lock at 1733761881422Disabling compacts and flushes for region at 1733761881422Disabling writes for close at 1733761881422Writing region close event to WAL at 1733761881443 (+21 ms)Closed at 1733761881443 2024-12-09T16:31:21,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741927_1106 (size=2805) 2024-12-09T16:31:21,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741927_1106 (size=2805) 2024-12-09T16:31:21,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741927_1106 (size=2805) 2024-12-09T16:31:21,484 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881186, size=2.7 K (2805bytes) 2024-12-09T16:31:21,484 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881186 2024-12-09T16:31:21,485 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881186 after 1ms 2024-12-09T16:31:21,490 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881186: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:31:21,490 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881186 took 6ms 2024-12-09T16:31:21,500 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881186 so closing down 2024-12-09T16:31:21,500 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T16:31:21,505 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733761881186.temp 2024-12-09T16:31:21,514 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000003-wal.1733761881186.temp 2024-12-09T16:31:21,514 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T16:31:21,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741929_1108 (size=2312) 2024-12-09T16:31:21,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741929_1108 (size=2312) 2024-12-09T16:31:21,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741929_1108 (size=2312) 2024-12-09T16:31:21,544 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000003-wal.1733761881186.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-09T16:31:21,550 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000003-wal.1733761881186.temp to hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000035 2024-12-09T16:31:21,551 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 53 ms; skipped=2; WAL=hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881186, size=2.7 K, length=2805, corrupted=false, cancelled=false 2024-12-09T16:31:21,551 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881186, journal: Splitting hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881186, size=2.7 K (2805bytes) at 1733761881484Finishing writing output for hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881186 so closing down at 1733761881500 (+16 ms)Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000003-wal.1733761881186.temp at 1733761881514 (+14 ms)3 split writer threads finished at 1733761881514Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000003-wal.1733761881186.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733761881544 (+30 ms)Rename recovered edits hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000003-wal.1733761881186.temp to hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000035 at 1733761881550 (+6 ms)Processed 32 edits across 1 Regions in 53 ms; skipped=2; WAL=hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881186, size=2.7 K, length=2805, corrupted=false, cancelled=false at 1733761881551 (+1 ms) 2024-12-09T16:31:21,560 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881186 to hdfs://localhost:39607/hbase/oldWALs/wal.1733761881186 2024-12-09T16:31:21,563 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000035 2024-12-09T16:31:21,563 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T16:31:21,566 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:31:21,587 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881567, exclude list is [], retry=0 2024-12-09T16:31:21,592 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:31:21,593 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:31:21,593 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:31:21,606 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881567 2024-12-09T16:31:21,607 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39661:39661),(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:35519:35519)] 2024-12-09T16:31:21,607 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d14c45326746921f20d52909371268eb, NAME => 'testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb.', STARTKEY => '', ENDKEY => ''} 2024-12-09T16:31:21,607 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:31:21,607 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,608 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,630 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,632 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d14c45326746921f20d52909371268eb columnFamilyName a 2024-12-09T16:31:21,632 DEBUG [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:21,656 DEBUG [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/a/4ca437f65bd14feba5b7989d11c9516c 2024-12-09T16:31:21,656 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.HStore(327): Store=d14c45326746921f20d52909371268eb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:21,658 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,659 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d14c45326746921f20d52909371268eb columnFamilyName b 2024-12-09T16:31:21,659 DEBUG [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:21,661 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.HStore(327): Store=d14c45326746921f20d52909371268eb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:21,661 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,662 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d14c45326746921f20d52909371268eb columnFamilyName c 2024-12-09T16:31:21,663 DEBUG [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:21,663 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.HStore(327): Store=d14c45326746921f20d52909371268eb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:21,664 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,665 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,668 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb 2024-12-09T16:31:21,669 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000035 2024-12-09T16:31:21,674 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:31:21,676 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000035 2024-12-09T16:31:21,676 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d14c45326746921f20d52909371268eb 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-09T16:31:21,701 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/.tmp/b/a474150c34a44a529c3fb7bbcee64b0d is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733761881374/Put/seqid=0 2024-12-09T16:31:21,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741931_1110 (size=5958) 2024-12-09T16:31:21,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741931_1110 (size=5958) 2024-12-09T16:31:21,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741931_1110 (size=5958) 2024-12-09T16:31:22,119 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/.tmp/b/a474150c34a44a529c3fb7bbcee64b0d 2024-12-09T16:31:22,142 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/.tmp/c/cf311fa849534537bb2167630d03d415 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733761881388/Put/seqid=0 2024-12-09T16:31:22,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741932_1111 (size=5958) 2024-12-09T16:31:22,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741932_1111 (size=5958) 2024-12-09T16:31:22,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741932_1111 (size=5958) 2024-12-09T16:31:22,558 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/.tmp/c/cf311fa849534537bb2167630d03d415 2024-12-09T16:31:22,568 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/.tmp/b/a474150c34a44a529c3fb7bbcee64b0d as hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/b/a474150c34a44a529c3fb7bbcee64b0d 2024-12-09T16:31:22,582 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/b/a474150c34a44a529c3fb7bbcee64b0d, entries=10, sequenceid=35, filesize=5.8 K 2024-12-09T16:31:22,584 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/.tmp/c/cf311fa849534537bb2167630d03d415 as hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/c/cf311fa849534537bb2167630d03d415 2024-12-09T16:31:22,597 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/c/cf311fa849534537bb2167630d03d415, entries=10, sequenceid=35, filesize=5.8 K 2024-12-09T16:31:22,598 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for d14c45326746921f20d52909371268eb in 922ms, sequenceid=35, compaction requested=false; wal=null 2024-12-09T16:31:22,659 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000035 2024-12-09T16:31:22,661 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d14c45326746921f20d52909371268eb 2024-12-09T16:31:22,661 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d14c45326746921f20d52909371268eb 2024-12-09T16:31:22,664 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T16:31:22,667 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d14c45326746921f20d52909371268eb 2024-12-09T16:31:22,671 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-09T16:31:22,672 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d14c45326746921f20d52909371268eb; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62128592, jitterRate=-0.07421183586120605}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T16:31:22,673 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d14c45326746921f20d52909371268eb: Writing region info on filesystem at 1733761881608Initializing all the Stores at 1733761881614 (+6 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761881614Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761881627 (+13 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761881627Obtaining lock to block concurrent updates at 1733761881676 (+49 ms)Preparing flush snapshotting stores in d14c45326746921f20d52909371268eb at 1733761881676Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733761881677 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb. at 1733761881677Flushing d14c45326746921f20d52909371268eb/b: creating writer at 1733761881677Flushing d14c45326746921f20d52909371268eb/b: appending metadata at 1733761881700 (+23 ms)Flushing d14c45326746921f20d52909371268eb/b: closing flushed file at 1733761881700Flushing d14c45326746921f20d52909371268eb/c: creating writer at 1733761882127 (+427 ms)Flushing d14c45326746921f20d52909371268eb/c: appending metadata at 1733761882142 (+15 ms)Flushing d14c45326746921f20d52909371268eb/c: closing flushed file at 1733761882142Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@151cfd80: reopening flushed file at 1733761882567 (+425 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63cc261a: reopening flushed file at 1733761882582 (+15 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for d14c45326746921f20d52909371268eb in 922ms, sequenceid=35, compaction requested=false; wal=null at 1733761882655 (+73 ms)Cleaning up temporary data from old regions at 1733761882661 (+6 ms)Region opened successfully at 1733761882672 (+11 ms) 2024-12-09T16:31:22,790 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881567, size=0 (0bytes) 2024-12-09T16:31:22,790 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881567 might be still open, length is 0 2024-12-09T16:31:22,790 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881567 2024-12-09T16:31:22,791 WARN [IPC Server handler 1 on default port 39607 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881567 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-12-09T16:31:22,791 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881567 after 1ms 2024-12-09T16:31:23,796 INFO [master/dfce6ce5b10c:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-09T16:31:23,796 INFO [master/dfce6ce5b10c:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-09T16:31:23,928 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:60692 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:39721:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60692 dst: /127.0.0.1:39721 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39721 remote=/127.0.0.1:60692]. Total timeout mills is 60000, 58802 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:31:23,930 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:51430 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:41451:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51430 dst: /127.0.0.1:41451 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:31:23,930 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-922664622_22 at /127.0.0.1:51374 [Receiving block BP-427817209-172.17.0.2-1733761835089:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:36895:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51374 dst: /127.0.0.1:36895 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:31:23,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741930_1112 (size=2304) 2024-12-09T16:31:23,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741930_1112 (size=2304) 2024-12-09T16:31:26,792 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881567 after 4002ms 2024-12-09T16:31:26,796 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881567: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:31:26,796 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881567 took 4006ms 2024-12-09T16:31:26,800 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881567; continuing. 2024-12-09T16:31:26,800 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881567 so closing down 2024-12-09T16:31:26,800 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T16:31:26,802 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733761881567.temp 2024-12-09T16:31:26,804 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000037-wal.1733761881567.temp 2024-12-09T16:31:26,804 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T16:31:26,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741933_1113 (size=2312) 2024-12-09T16:31:26,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741933_1113 (size=2312) 2024-12-09T16:31:26,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741933_1113 (size=2312) 2024-12-09T16:31:26,828 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000037-wal.1733761881567.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-09T16:31:26,830 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000037-wal.1733761881567.temp to hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000066 2024-12-09T16:31:26,830 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 32 ms; skipped=0; WAL=hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881567, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T16:31:26,830 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881567, journal: Splitting hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881567, size=0 (0bytes) at 1733761882790Finishing writing output for hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881567 so closing down at 1733761886800 (+4010 ms)Creating recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000037-wal.1733761881567.temp at 1733761886804 (+4 ms)3 split writer threads finished at 1733761886804Closed recovered edits writer path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000037-wal.1733761881567.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733761886828 (+24 ms)Rename recovered edits hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000037-wal.1733761881567.temp to hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000066 at 1733761886830 (+2 ms)Processed 30 edits across 1 Regions in 32 ms; skipped=0; WAL=hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881567, size=0, length=0, corrupted=false, cancelled=false at 1733761886830 2024-12-09T16:31:26,833 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881567 to hdfs://localhost:39607/hbase/oldWALs/wal.1733761881567 2024-12-09T16:31:26,834 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000066 2024-12-09T16:31:26,834 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T16:31:26,838 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39607/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048, archiveDir=hdfs://localhost:39607/hbase/oldWALs, maxLogs=32 2024-12-09T16:31:26,860 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761886839, exclude list is [], retry=0 2024-12-09T16:31:26,864 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39721,DS-988cafaf-517a-41ae-b218-7e4ffd5ee141,DISK] 2024-12-09T16:31:26,864 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41451,DS-b118a02d-bd6c-42db-be6e-e730304ac329,DISK] 2024-12-09T16:31:26,865 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36895,DS-36c2801e-54a9-44e5-a948-0b1b1a47424d,DISK] 2024-12-09T16:31:26,870 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761886839 2024-12-09T16:31:26,871 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39661:39661),(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:35519:35519)] 2024-12-09T16:31:26,871 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T16:31:26,874 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d14c45326746921f20d52909371268eb 2024-12-09T16:31:26,875 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d14c45326746921f20d52909371268eb columnFamilyName a 2024-12-09T16:31:26,875 DEBUG [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:26,884 DEBUG [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/a/4ca437f65bd14feba5b7989d11c9516c 2024-12-09T16:31:26,884 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.HStore(327): Store=d14c45326746921f20d52909371268eb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:26,885 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d14c45326746921f20d52909371268eb 2024-12-09T16:31:26,886 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d14c45326746921f20d52909371268eb columnFamilyName b 2024-12-09T16:31:26,886 DEBUG [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:26,895 DEBUG [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/b/a474150c34a44a529c3fb7bbcee64b0d 2024-12-09T16:31:26,896 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.HStore(327): Store=d14c45326746921f20d52909371268eb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:26,896 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d14c45326746921f20d52909371268eb 2024-12-09T16:31:26,899 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d14c45326746921f20d52909371268eb columnFamilyName c 2024-12-09T16:31:26,899 DEBUG [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T16:31:26,904 DEBUG [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/c/cf311fa849534537bb2167630d03d415 2024-12-09T16:31:26,904 INFO [StoreOpener-d14c45326746921f20d52909371268eb-1 {}] regionserver.HStore(327): Store=d14c45326746921f20d52909371268eb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T16:31:26,905 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d14c45326746921f20d52909371268eb 2024-12-09T16:31:26,906 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb 2024-12-09T16:31:26,907 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb 2024-12-09T16:31:26,908 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000066 2024-12-09T16:31:26,910 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-09T16:31:26,914 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000066 2024-12-09T16:31:26,914 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d14c45326746921f20d52909371268eb 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-09T16:31:26,934 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/.tmp/a/403fafdc04ad4567ab78a51e5ba59e78 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733761882685/Put/seqid=0 2024-12-09T16:31:26,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741935_1115 (size=5958) 2024-12-09T16:31:26,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741935_1115 (size=5958) 2024-12-09T16:31:26,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741935_1115 (size=5958) 2024-12-09T16:31:26,967 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/.tmp/a/403fafdc04ad4567ab78a51e5ba59e78 2024-12-09T16:31:26,990 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/.tmp/b/b2f18b6e4e5f4eb7b67d0539e52bd444 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733761882700/Put/seqid=0 2024-12-09T16:31:27,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741936_1116 (size=5958) 2024-12-09T16:31:27,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741936_1116 (size=5958) 2024-12-09T16:31:27,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741936_1116 (size=5958) 2024-12-09T16:31:27,015 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/.tmp/b/b2f18b6e4e5f4eb7b67d0539e52bd444 2024-12-09T16:31:27,078 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/.tmp/c/51aa0e7344314cde86f60fe9f82c9f04 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733761882717/Put/seqid=0 2024-12-09T16:31:27,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741937_1117 (size=5958) 2024-12-09T16:31:27,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741937_1117 (size=5958) 2024-12-09T16:31:27,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741937_1117 (size=5958) 2024-12-09T16:31:27,100 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/.tmp/c/51aa0e7344314cde86f60fe9f82c9f04 2024-12-09T16:31:27,112 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/.tmp/a/403fafdc04ad4567ab78a51e5ba59e78 as hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/a/403fafdc04ad4567ab78a51e5ba59e78 2024-12-09T16:31:27,120 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/a/403fafdc04ad4567ab78a51e5ba59e78, entries=10, sequenceid=66, filesize=5.8 K 2024-12-09T16:31:27,122 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/.tmp/b/b2f18b6e4e5f4eb7b67d0539e52bd444 as hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/b/b2f18b6e4e5f4eb7b67d0539e52bd444 2024-12-09T16:31:27,129 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/b/b2f18b6e4e5f4eb7b67d0539e52bd444, entries=10, sequenceid=66, filesize=5.8 K 2024-12-09T16:31:27,130 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/.tmp/c/51aa0e7344314cde86f60fe9f82c9f04 as hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/c/51aa0e7344314cde86f60fe9f82c9f04 2024-12-09T16:31:27,150 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/c/51aa0e7344314cde86f60fe9f82c9f04, entries=10, sequenceid=66, filesize=5.8 K 2024-12-09T16:31:27,150 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for d14c45326746921f20d52909371268eb in 236ms, sequenceid=66, compaction requested=false; wal=null 2024-12-09T16:31:27,151 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/0000000000000000066 2024-12-09T16:31:27,153 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d14c45326746921f20d52909371268eb 2024-12-09T16:31:27,153 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d14c45326746921f20d52909371268eb 2024-12-09T16:31:27,154 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T16:31:27,157 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d14c45326746921f20d52909371268eb 2024-12-09T16:31:27,166 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/hbase/data/default/testReplayEditsWrittenViaHRegion/d14c45326746921f20d52909371268eb/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-09T16:31:27,167 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d14c45326746921f20d52909371268eb; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71939045, jitterRate=0.07197530567646027}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T16:31:27,168 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d14c45326746921f20d52909371268eb: Writing region info on filesystem at 1733761886872Initializing all the Stores at 1733761886873 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761886873Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761886873Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733761886873Obtaining lock to block concurrent updates at 1733761886914 (+41 ms)Preparing flush snapshotting stores in d14c45326746921f20d52909371268eb at 1733761886914Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733761886914Flushing stores of testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb. at 1733761886914Flushing d14c45326746921f20d52909371268eb/a: creating writer at 1733761886914Flushing d14c45326746921f20d52909371268eb/a: appending metadata at 1733761886933 (+19 ms)Flushing d14c45326746921f20d52909371268eb/a: closing flushed file at 1733761886933Flushing d14c45326746921f20d52909371268eb/b: creating writer at 1733761886973 (+40 ms)Flushing d14c45326746921f20d52909371268eb/b: appending metadata at 1733761886989 (+16 ms)Flushing d14c45326746921f20d52909371268eb/b: closing flushed file at 1733761886989Flushing d14c45326746921f20d52909371268eb/c: creating writer at 1733761887022 (+33 ms)Flushing d14c45326746921f20d52909371268eb/c: appending metadata at 1733761887075 (+53 ms)Flushing d14c45326746921f20d52909371268eb/c: closing flushed file at 1733761887075Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ba738f8: reopening flushed file at 1733761887111 (+36 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@108ed21e: reopening flushed file at 1733761887120 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49d5f06a: reopening flushed file at 1733761887129 (+9 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for d14c45326746921f20d52909371268eb in 236ms, sequenceid=66, compaction requested=false; wal=null at 1733761887150 (+21 ms)Cleaning up temporary data from old regions at 1733761887153 (+3 ms)Region opened successfully at 1733761887168 (+15 ms) 2024-12-09T16:31:27,188 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d14c45326746921f20d52909371268eb, disabling compactions & flushes 2024-12-09T16:31:27,188 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb. 2024-12-09T16:31:27,188 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb. 2024-12-09T16:31:27,188 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb. after waiting 0 ms 2024-12-09T16:31:27,188 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb. 2024-12-09T16:31:27,192 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733761881049.d14c45326746921f20d52909371268eb. 2024-12-09T16:31:27,192 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d14c45326746921f20d52909371268eb: Waiting for close lock at 1733761887188Disabling compacts and flushes for region at 1733761887188Disabling writes for close at 1733761887188Writing region close event to WAL at 1733761887192 (+4 ms)Closed at 1733761887192 2024-12-09T16:31:27,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741934_1114 (size=93) 2024-12-09T16:31:27,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741934_1114 (size=93) 2024-12-09T16:31:27,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741934_1114 (size=93) 2024-12-09T16:31:27,201 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T16:31:27,201 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733761886839) 2024-12-09T16:31:27,225 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=448 (was 443) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1037739564_22 at /127.0.0.1:36138 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1037739564_22 at /127.0.0.1:32796 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:39607 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:39607 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1037739564_22 at /127.0.0.1:57132 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1408 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=593 (was 601), ProcessCount=11 (was 11), AvailableMemoryMB=7179 (was 7287) 2024-12-09T16:31:27,225 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1408 is superior to 1024 2024-12-09T16:31:27,225 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-09T16:31:27,225 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T16:31:27,226 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T16:31:27,226 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T16:31:27,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T16:31:27,227 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T16:31:27,227 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T16:31:27,227 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1849544579, stopped=false 2024-12-09T16:31:27,228 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=dfce6ce5b10c,35827,1733761839880 2024-12-09T16:31:27,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T16:31:27,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38861-0x100bfc42e970001, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T16:31:27,266 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34709-0x100bfc42e970003, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T16:31:27,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38861-0x100bfc42e970001, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:31:27,267 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T16:31:27,268 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38861-0x100bfc42e970001, quorum=127.0.0.1:60962, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T16:31:27,268 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T16:31:27,269 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34709-0x100bfc42e970003, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:31:27,269 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T16:31:27,269 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T16:31:27,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:31:27,269 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T16:31:27,270 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dfce6ce5b10c,38861,1733761840646' ***** 2024-12-09T16:31:27,270 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T16:31:27,271 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dfce6ce5b10c,34709,1733761840861' ***** 2024-12-09T16:31:27,271 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34709-0x100bfc42e970003, quorum=127.0.0.1:60962, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T16:31:27,271 INFO [RS:0;dfce6ce5b10c:38861 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T16:31:27,271 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T16:31:27,271 INFO [RS:0;dfce6ce5b10c:38861 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T16:31:27,271 INFO [RS:0;dfce6ce5b10c:38861 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T16:31:27,271 INFO [RS:2;dfce6ce5b10c:34709 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T16:31:27,271 INFO [RS:0;dfce6ce5b10c:38861 {}] regionserver.HRegionServer(959): stopping server dfce6ce5b10c,38861,1733761840646 2024-12-09T16:31:27,271 INFO [RS:2;dfce6ce5b10c:34709 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T16:31:27,271 INFO [RS:0;dfce6ce5b10c:38861 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T16:31:27,271 INFO [RS:2;dfce6ce5b10c:34709 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T16:31:27,271 INFO [RS:0;dfce6ce5b10c:38861 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;dfce6ce5b10c:38861. 2024-12-09T16:31:27,271 INFO [RS:2;dfce6ce5b10c:34709 {}] regionserver.HRegionServer(3091): Received CLOSE for f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:31:27,271 DEBUG [RS:0;dfce6ce5b10c:38861 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T16:31:27,271 DEBUG [RS:0;dfce6ce5b10c:38861 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T16:31:27,271 INFO [RS:0;dfce6ce5b10c:38861 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T16:31:27,271 INFO [RS:0;dfce6ce5b10c:38861 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T16:31:27,271 INFO [RS:0;dfce6ce5b10c:38861 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T16:31:27,272 INFO [RS:0;dfce6ce5b10c:38861 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T16:31:27,272 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T16:31:27,272 INFO [RS:2;dfce6ce5b10c:34709 {}] regionserver.HRegionServer(959): stopping server dfce6ce5b10c,34709,1733761840861 2024-12-09T16:31:27,272 INFO [RS:2;dfce6ce5b10c:34709 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T16:31:27,272 INFO [RS:2;dfce6ce5b10c:34709 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;dfce6ce5b10c:34709. 2024-12-09T16:31:27,272 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f010e6e532012a33c4c8a2a892bd1818, disabling compactions & flushes 2024-12-09T16:31:27,272 DEBUG [RS:2;dfce6ce5b10c:34709 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T16:31:27,272 DEBUG [RS:2;dfce6ce5b10c:34709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T16:31:27,272 INFO [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:31:27,271 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T16:31:27,272 INFO [RS:2;dfce6ce5b10c:34709 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T16:31:27,272 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:31:27,272 DEBUG [RS:2;dfce6ce5b10c:34709 {}] regionserver.HRegionServer(1325): Online Regions={f010e6e532012a33c4c8a2a892bd1818=testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818.} 2024-12-09T16:31:27,272 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. after waiting 0 ms 2024-12-09T16:31:27,272 DEBUG [RS:2;dfce6ce5b10c:34709 {}] regionserver.HRegionServer(1351): Waiting on f010e6e532012a33c4c8a2a892bd1818 2024-12-09T16:31:27,272 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:31:27,273 INFO [RS:0;dfce6ce5b10c:38861 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T16:31:27,273 DEBUG [RS:0;dfce6ce5b10c:38861 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-09T16:31:27,273 DEBUG [RS:0;dfce6ce5b10c:38861 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T16:31:27,273 DEBUG [RS_CLOSE_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T16:31:27,274 INFO [RS_CLOSE_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T16:31:27,274 DEBUG [RS_CLOSE_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T16:31:27,274 DEBUG [RS_CLOSE_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T16:31:27,274 DEBUG [RS_CLOSE_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T16:31:27,274 INFO [RS_CLOSE_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-12-09T16:31:27,320 DEBUG [RS_CLOSE_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/hbase/meta/1588230740/.tmp/info/10334360486d4503baf4974417ccb6c3 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818./info:regioninfo/1733761861117/Put/seqid=0 2024-12-09T16:31:27,321 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/default/testReplayEditsAfterRegionMovedWithMultiCF/f010e6e532012a33c4c8a2a892bd1818/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-09T16:31:27,323 INFO [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:31:27,323 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f010e6e532012a33c4c8a2a892bd1818: Waiting for close lock at 1733761887272Running coprocessor pre-close hooks at 1733761887272Disabling compacts and flushes for region at 1733761887272Disabling writes for close at 1733761887272Writing region close event to WAL at 1733761887277 (+5 ms)Running coprocessor post-close hooks at 1733761887323 (+46 ms)Closed at 1733761887323 2024-12-09T16:31:27,323 DEBUG [RS_CLOSE_REGION-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733761857712.f010e6e532012a33c4c8a2a892bd1818. 2024-12-09T16:31:27,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741938_1118 (size=8243) 2024-12-09T16:31:27,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741938_1118 (size=8243) 2024-12-09T16:31:27,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741938_1118 (size=8243) 2024-12-09T16:31:27,360 INFO [regionserver/dfce6ce5b10c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T16:31:27,360 INFO [regionserver/dfce6ce5b10c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T16:31:27,419 INFO [regionserver/dfce6ce5b10c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-09T16:31:27,419 INFO [regionserver/dfce6ce5b10c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-09T16:31:27,449 INFO [regionserver/dfce6ce5b10c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-09T16:31:27,449 INFO [regionserver/dfce6ce5b10c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-09T16:31:27,473 INFO [RS:2;dfce6ce5b10c:34709 {}] regionserver.HRegionServer(976): stopping server dfce6ce5b10c,34709,1733761840861; all regions closed. 2024-12-09T16:31:27,473 DEBUG [RS:0;dfce6ce5b10c:38861 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T16:31:27,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741835_1011 (size=841) 2024-12-09T16:31:27,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741835_1011 (size=841) 2024-12-09T16:31:27,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741835_1011 (size=841) 2024-12-09T16:31:27,481 DEBUG [RS:2;dfce6ce5b10c:34709 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/oldWALs 2024-12-09T16:31:27,481 INFO [RS:2;dfce6ce5b10c:34709 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL dfce6ce5b10c%2C34709%2C1733761840861:(num 1733761842531) 2024-12-09T16:31:27,481 DEBUG [RS:2;dfce6ce5b10c:34709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T16:31:27,481 INFO [RS:2;dfce6ce5b10c:34709 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T16:31:27,481 INFO [RS:2;dfce6ce5b10c:34709 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T16:31:27,482 INFO [RS:2;dfce6ce5b10c:34709 {}] hbase.ChoreService(370): Chore service for: regionserver/dfce6ce5b10c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T16:31:27,482 INFO [RS:2;dfce6ce5b10c:34709 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T16:31:27,482 INFO [RS:2;dfce6ce5b10c:34709 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T16:31:27,482 INFO [RS:2;dfce6ce5b10c:34709 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T16:31:27,482 INFO [RS:2;dfce6ce5b10c:34709 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T16:31:27,482 INFO [RS:2;dfce6ce5b10c:34709 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34709 2024-12-09T16:31:27,482 INFO [regionserver/dfce6ce5b10c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T16:31:27,491 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34709-0x100bfc42e970003, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dfce6ce5b10c,34709,1733761840861 2024-12-09T16:31:27,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T16:31:27,491 INFO [RS:2;dfce6ce5b10c:34709 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T16:31:27,492 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dfce6ce5b10c,34709,1733761840861] 2024-12-09T16:31:27,507 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dfce6ce5b10c,34709,1733761840861 already deleted, retry=false 2024-12-09T16:31:27,508 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dfce6ce5b10c,34709,1733761840861 expired; onlineServers=1 2024-12-09T16:31:27,600 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34709-0x100bfc42e970003, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T16:31:27,600 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34709-0x100bfc42e970003, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T16:31:27,600 INFO [RS:2;dfce6ce5b10c:34709 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T16:31:27,600 INFO [RS:2;dfce6ce5b10c:34709 {}] regionserver.HRegionServer(1031): Exiting; stopping=dfce6ce5b10c,34709,1733761840861; zookeeper connection closed. 2024-12-09T16:31:27,600 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3dc50b64 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3dc50b64 2024-12-09T16:31:27,673 DEBUG [RS:0;dfce6ce5b10c:38861 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T16:31:27,739 INFO [RS_CLOSE_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/hbase/meta/1588230740/.tmp/info/10334360486d4503baf4974417ccb6c3 2024-12-09T16:31:27,770 DEBUG [RS_CLOSE_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/hbase/meta/1588230740/.tmp/ns/65e360dd34584c7d82283a56362b137d is 43, key is default/ns:d/1733761843667/Put/seqid=0 2024-12-09T16:31:27,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741939_1119 (size=5153) 2024-12-09T16:31:27,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741939_1119 (size=5153) 2024-12-09T16:31:27,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741939_1119 (size=5153) 2024-12-09T16:31:27,781 INFO [RS_CLOSE_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/hbase/meta/1588230740/.tmp/ns/65e360dd34584c7d82283a56362b137d 2024-12-09T16:31:27,818 DEBUG [RS_CLOSE_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/hbase/meta/1588230740/.tmp/table/9a1baf5787054f29a1c09efa3aa13b01 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733761858150/Put/seqid=0 2024-12-09T16:31:27,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741940_1120 (size=5431) 2024-12-09T16:31:27,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741940_1120 (size=5431) 2024-12-09T16:31:27,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741940_1120 (size=5431) 2024-12-09T16:31:27,840 INFO [RS_CLOSE_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/hbase/meta/1588230740/.tmp/table/9a1baf5787054f29a1c09efa3aa13b01 2024-12-09T16:31:27,853 DEBUG [RS_CLOSE_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/hbase/meta/1588230740/.tmp/info/10334360486d4503baf4974417ccb6c3 as hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/hbase/meta/1588230740/info/10334360486d4503baf4974417ccb6c3 2024-12-09T16:31:27,862 INFO [RS_CLOSE_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/hbase/meta/1588230740/info/10334360486d4503baf4974417ccb6c3, entries=18, sequenceid=21, filesize=8.0 K 2024-12-09T16:31:27,864 DEBUG [RS_CLOSE_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/hbase/meta/1588230740/.tmp/ns/65e360dd34584c7d82283a56362b137d as hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/hbase/meta/1588230740/ns/65e360dd34584c7d82283a56362b137d 2024-12-09T16:31:27,874 DEBUG [RS:0;dfce6ce5b10c:38861 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T16:31:27,874 INFO [RS_CLOSE_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/hbase/meta/1588230740/ns/65e360dd34584c7d82283a56362b137d, entries=2, sequenceid=21, filesize=5.0 K 2024-12-09T16:31:27,877 DEBUG [RS_CLOSE_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/hbase/meta/1588230740/.tmp/table/9a1baf5787054f29a1c09efa3aa13b01 as hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/hbase/meta/1588230740/table/9a1baf5787054f29a1c09efa3aa13b01 2024-12-09T16:31:27,888 INFO [RS_CLOSE_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/hbase/meta/1588230740/table/9a1baf5787054f29a1c09efa3aa13b01, entries=2, sequenceid=21, filesize=5.3 K 2024-12-09T16:31:27,890 INFO [RS_CLOSE_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 616ms, sequenceid=21, compaction requested=false 2024-12-09T16:31:27,901 DEBUG [RS_CLOSE_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-12-09T16:31:27,902 DEBUG [RS_CLOSE_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T16:31:27,902 INFO [RS_CLOSE_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T16:31:27,902 DEBUG [RS_CLOSE_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733761887273Running coprocessor pre-close hooks at 1733761887273Disabling compacts and flushes for region at 1733761887273Disabling writes for close at 1733761887274 (+1 ms)Obtaining lock to block concurrent updates at 1733761887274Preparing flush snapshotting stores in 1588230740 at 1733761887274Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1733761887275 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733761887276 (+1 ms)Flushing 1588230740/info: creating writer at 1733761887276Flushing 1588230740/info: appending metadata at 1733761887319 (+43 ms)Flushing 1588230740/info: closing flushed file at 1733761887319Flushing 1588230740/ns: creating writer at 1733761887747 (+428 ms)Flushing 1588230740/ns: appending metadata at 1733761887769 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1733761887769Flushing 1588230740/table: creating writer at 1733761887790 (+21 ms)Flushing 1588230740/table: appending metadata at 1733761887817 (+27 ms)Flushing 1588230740/table: closing flushed file at 1733761887817Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1149041b: reopening flushed file at 1733761887851 (+34 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d2cfba7: reopening flushed file at 1733761887863 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61f846c5: reopening flushed file at 1733761887874 (+11 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 616ms, sequenceid=21, compaction requested=false at 1733761887890 (+16 ms)Writing region close event to WAL at 1733761887894 (+4 ms)Running coprocessor post-close hooks at 1733761887902 (+8 ms)Closed at 1733761887902 2024-12-09T16:31:27,903 DEBUG [RS_CLOSE_META-regionserver/dfce6ce5b10c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T16:31:28,074 INFO [RS:0;dfce6ce5b10c:38861 {}] regionserver.HRegionServer(976): stopping server dfce6ce5b10c,38861,1733761840646; all regions closed. 2024-12-09T16:31:28,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741836_1012 (size=3783) 2024-12-09T16:31:28,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741836_1012 (size=3783) 2024-12-09T16:31:28,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741836_1012 (size=3783) 2024-12-09T16:31:28,085 DEBUG [RS:0;dfce6ce5b10c:38861 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/oldWALs 2024-12-09T16:31:28,085 INFO [RS:0;dfce6ce5b10c:38861 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL dfce6ce5b10c%2C38861%2C1733761840646.meta:.meta(num 1733761843203) 2024-12-09T16:31:28,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741833_1009 (size=2054) 2024-12-09T16:31:28,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741833_1009 (size=2054) 2024-12-09T16:31:28,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741833_1009 (size=2054) 2024-12-09T16:31:28,099 DEBUG [RS:0;dfce6ce5b10c:38861 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/oldWALs 2024-12-09T16:31:28,099 INFO [RS:0;dfce6ce5b10c:38861 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL dfce6ce5b10c%2C38861%2C1733761840646:(num 1733761842531) 2024-12-09T16:31:28,099 DEBUG [RS:0;dfce6ce5b10c:38861 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T16:31:28,099 INFO [RS:0;dfce6ce5b10c:38861 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T16:31:28,099 INFO [RS:0;dfce6ce5b10c:38861 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T16:31:28,099 INFO [RS:0;dfce6ce5b10c:38861 {}] hbase.ChoreService(370): Chore service for: regionserver/dfce6ce5b10c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T16:31:28,100 INFO [RS:0;dfce6ce5b10c:38861 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T16:31:28,100 INFO [regionserver/dfce6ce5b10c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T16:31:28,100 INFO [RS:0;dfce6ce5b10c:38861 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38861 2024-12-09T16:31:28,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T16:31:28,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38861-0x100bfc42e970001, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dfce6ce5b10c,38861,1733761840646 2024-12-09T16:31:28,116 INFO [RS:0;dfce6ce5b10c:38861 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T16:31:28,117 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dfce6ce5b10c,38861,1733761840646] 2024-12-09T16:31:28,132 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dfce6ce5b10c,38861,1733761840646 already deleted, retry=false 2024-12-09T16:31:28,133 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dfce6ce5b10c,38861,1733761840646 expired; onlineServers=0 2024-12-09T16:31:28,133 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'dfce6ce5b10c,35827,1733761839880' ***** 2024-12-09T16:31:28,133 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-09T16:31:28,133 INFO [M:0;dfce6ce5b10c:35827 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T16:31:28,133 INFO [M:0;dfce6ce5b10c:35827 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T16:31:28,133 DEBUG [M:0;dfce6ce5b10c:35827 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T16:31:28,134 DEBUG [M:0;dfce6ce5b10c:35827 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T16:31:28,134 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T16:31:28,134 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster-HFileCleaner.small.0-1733761842110 {}] cleaner.HFileCleaner(306): Exit Thread[master/dfce6ce5b10c:0:becomeActiveMaster-HFileCleaner.small.0-1733761842110,5,FailOnTimeoutGroup] 2024-12-09T16:31:28,134 DEBUG [master/dfce6ce5b10c:0:becomeActiveMaster-HFileCleaner.large.0-1733761842101 {}] cleaner.HFileCleaner(306): Exit Thread[master/dfce6ce5b10c:0:becomeActiveMaster-HFileCleaner.large.0-1733761842101,5,FailOnTimeoutGroup] 2024-12-09T16:31:28,134 INFO [M:0;dfce6ce5b10c:35827 {}] hbase.ChoreService(370): Chore service for: master/dfce6ce5b10c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-09T16:31:28,134 INFO [M:0;dfce6ce5b10c:35827 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T16:31:28,134 DEBUG [M:0;dfce6ce5b10c:35827 {}] master.HMaster(1795): Stopping service threads 2024-12-09T16:31:28,134 INFO [M:0;dfce6ce5b10c:35827 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T16:31:28,134 INFO [M:0;dfce6ce5b10c:35827 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T16:31:28,135 INFO [M:0;dfce6ce5b10c:35827 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T16:31:28,136 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T16:31:28,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T16:31:28,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T16:31:28,141 DEBUG [M:0;dfce6ce5b10c:35827 {}] zookeeper.ZKUtil(347): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-09T16:31:28,141 WARN [M:0;dfce6ce5b10c:35827 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-09T16:31:28,142 INFO [M:0;dfce6ce5b10c:35827 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/.lastflushedseqids 2024-12-09T16:31:28,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741941_1121 (size=138) 2024-12-09T16:31:28,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741941_1121 (size=138) 2024-12-09T16:31:28,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741941_1121 (size=138) 2024-12-09T16:31:28,167 INFO [M:0;dfce6ce5b10c:35827 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T16:31:28,167 INFO [M:0;dfce6ce5b10c:35827 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T16:31:28,167 DEBUG [M:0;dfce6ce5b10c:35827 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T16:31:28,167 INFO [M:0;dfce6ce5b10c:35827 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T16:31:28,167 DEBUG [M:0;dfce6ce5b10c:35827 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T16:31:28,167 DEBUG [M:0;dfce6ce5b10c:35827 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T16:31:28,167 DEBUG [M:0;dfce6ce5b10c:35827 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T16:31:28,167 INFO [M:0;dfce6ce5b10c:35827 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-12-09T16:31:28,192 DEBUG [M:0;dfce6ce5b10c:35827 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aaddb7fb044a4eb78ef0d03a80604c16 is 82, key is hbase:meta,,1/info:regioninfo/1733761843307/Put/seqid=0 2024-12-09T16:31:28,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741942_1122 (size=5672) 2024-12-09T16:31:28,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741942_1122 (size=5672) 2024-12-09T16:31:28,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741942_1122 (size=5672) 2024-12-09T16:31:28,210 INFO [M:0;dfce6ce5b10c:35827 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aaddb7fb044a4eb78ef0d03a80604c16 2024-12-09T16:31:28,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38861-0x100bfc42e970001, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T16:31:28,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38861-0x100bfc42e970001, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T16:31:28,225 INFO [RS:0;dfce6ce5b10c:38861 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T16:31:28,225 INFO [RS:0;dfce6ce5b10c:38861 {}] regionserver.HRegionServer(1031): Exiting; stopping=dfce6ce5b10c,38861,1733761840646; zookeeper connection closed. 2024-12-09T16:31:28,230 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2e45fe8d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2e45fe8d 2024-12-09T16:31:28,234 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-09T16:31:28,240 DEBUG [M:0;dfce6ce5b10c:35827 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d97c4cf0709944c696517952eed27c5f is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733761858157/Put/seqid=0 2024-12-09T16:31:28,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741943_1123 (size=7755) 2024-12-09T16:31:28,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741943_1123 (size=7755) 2024-12-09T16:31:28,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741943_1123 (size=7755) 2024-12-09T16:31:28,666 INFO [M:0;dfce6ce5b10c:35827 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d97c4cf0709944c696517952eed27c5f 2024-12-09T16:31:28,673 INFO [M:0;dfce6ce5b10c:35827 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d97c4cf0709944c696517952eed27c5f 2024-12-09T16:31:28,696 DEBUG [M:0;dfce6ce5b10c:35827 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f4f6181106d84dce8afa5d3ee3749603 is 69, key is dfce6ce5b10c,34709,1733761840861/rs:state/1733761842215/Put/seqid=0 2024-12-09T16:31:28,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741944_1124 (size=5440) 2024-12-09T16:31:28,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741944_1124 (size=5440) 2024-12-09T16:31:28,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741944_1124 (size=5440) 2024-12-09T16:31:28,711 INFO [M:0;dfce6ce5b10c:35827 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f4f6181106d84dce8afa5d3ee3749603 2024-12-09T16:31:28,723 INFO [M:0;dfce6ce5b10c:35827 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f4f6181106d84dce8afa5d3ee3749603 2024-12-09T16:31:28,725 DEBUG [M:0;dfce6ce5b10c:35827 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aaddb7fb044a4eb78ef0d03a80604c16 as hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/aaddb7fb044a4eb78ef0d03a80604c16 2024-12-09T16:31:28,734 INFO [M:0;dfce6ce5b10c:35827 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/aaddb7fb044a4eb78ef0d03a80604c16, entries=8, sequenceid=168, filesize=5.5 K 2024-12-09T16:31:28,736 DEBUG [M:0;dfce6ce5b10c:35827 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d97c4cf0709944c696517952eed27c5f as hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d97c4cf0709944c696517952eed27c5f 2024-12-09T16:31:28,747 INFO [M:0;dfce6ce5b10c:35827 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d97c4cf0709944c696517952eed27c5f 2024-12-09T16:31:28,747 INFO [M:0;dfce6ce5b10c:35827 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d97c4cf0709944c696517952eed27c5f, entries=17, sequenceid=168, filesize=7.6 K 2024-12-09T16:31:28,749 DEBUG [M:0;dfce6ce5b10c:35827 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f4f6181106d84dce8afa5d3ee3749603 as hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f4f6181106d84dce8afa5d3ee3749603 2024-12-09T16:31:28,756 INFO [M:0;dfce6ce5b10c:35827 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f4f6181106d84dce8afa5d3ee3749603 2024-12-09T16:31:28,757 INFO [M:0;dfce6ce5b10c:35827 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39607/user/jenkins/test-data/4d04b195-a319-3336-e522-eab218ce7b38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f4f6181106d84dce8afa5d3ee3749603, entries=3, sequenceid=168, filesize=5.3 K 2024-12-09T16:31:28,759 INFO [M:0;dfce6ce5b10c:35827 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 591ms, sequenceid=168, compaction requested=false 2024-12-09T16:31:28,765 INFO [M:0;dfce6ce5b10c:35827 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T16:31:28,766 DEBUG [M:0;dfce6ce5b10c:35827 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733761888167Disabling compacts and flushes for region at 1733761888167Disabling writes for close at 1733761888167Obtaining lock to block concurrent updates at 1733761888167Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733761888167Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1733761888168 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733761888169 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733761888169Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733761888192 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733761888192Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733761888218 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733761888239 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733761888239Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733761888673 (+434 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733761888695 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733761888695Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@416a095e: reopening flushed file at 1733761888723 (+28 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@547d1cf5: reopening flushed file at 1733761888734 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23e0dd96: reopening flushed file at 1733761888747 (+13 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 591ms, sequenceid=168, compaction requested=false at 1733761888759 (+12 ms)Writing region close event to WAL at 1733761888765 (+6 ms)Closed at 1733761888765 2024-12-09T16:31:28,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36895 is added to blk_1073741830_1006 (size=71337) 2024-12-09T16:31:28,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39721 is added to blk_1073741830_1006 (size=71337) 2024-12-09T16:31:28,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41451 is added to blk_1073741830_1006 (size=71337) 2024-12-09T16:31:28,774 INFO [M:0;dfce6ce5b10c:35827 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-09T16:31:28,775 INFO [M:0;dfce6ce5b10c:35827 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35827 2024-12-09T16:31:28,775 INFO [M:0;dfce6ce5b10c:35827 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T16:31:28,775 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T16:31:28,916 INFO [M:0;dfce6ce5b10c:35827 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T16:31:28,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T16:31:28,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35827-0x100bfc42e970000, quorum=127.0.0.1:60962, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T16:31:28,925 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733761861273/wal.1733761862107 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733761861273/wal.1733761862107 (inode 16655) Holder DFSClient_NONMAPREDUCE_-922664622_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733761861273/wal.1733761862107 (inode 16655) Holder DFSClient_NONMAPREDUCE_-922664622_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-09T16:31:28,926 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733761852710/wal.1733761853381 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:31:28,927 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733761881048/wal.1733761881567 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:31:28,933 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733761862274/wal.1733761872114 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733761862274/wal.1733761872114 (inode 16678) Holder DFSClient_NONMAPREDUCE_-922664622_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733761862274/wal.1733761872114 (inode 16678) Holder DFSClient_NONMAPREDUCE_-922664622_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-09T16:31:28,934 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733761844856/wal.1733761845050 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:31:28,939 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733761872880/wal.1733761874284 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733761872880/wal.1733761874284 (inode 16704) Holder DFSClient_NONMAPREDUCE_-922664622_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733761872880/wal.1733761874284 (inode 16704) Holder DFSClient_NONMAPREDUCE_-922664622_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-09T16:31:28,940 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733761874822/wal.1733761875019 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T16:31:28,943 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal.1733761844688 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal.1733761844688 (inode 16485) Holder DFSClient_NONMAPREDUCE_-922664622_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733761844455/wal.1733761844688 (inode 16485) Holder DFSClient_NONMAPREDUCE_-922664622_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-09T16:31:28,947 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal.1733761844175 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal.1733761844175 (inode 16462) Holder DFSClient_NONMAPREDUCE_-922664622_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733761843917/wal.1733761844175 (inode 16462) Holder DFSClient_NONMAPREDUCE_-922664622_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-09T16:31:28,960 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4290616c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T16:31:28,963 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5e25d2b6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T16:31:28,963 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T16:31:28,963 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6853f5bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T16:31:28,964 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1bf0fdb4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/hadoop.log.dir/,STOPPED} 2024-12-09T16:31:28,972 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T16:31:28,973 WARN [BP-427817209-172.17.0.2-1733761835089 heartbeating to localhost/127.0.0.1:39607 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T16:31:28,973 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T16:31:28,973 WARN [BP-427817209-172.17.0.2-1733761835089 heartbeating to localhost/127.0.0.1:39607 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-427817209-172.17.0.2-1733761835089 (Datanode Uuid 1ef52798-38bb-448b-afff-499d10e83cc1) service to localhost/127.0.0.1:39607 2024-12-09T16:31:28,974 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T16:31:28,976 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/cluster_cd98e670-d246-b6f3-ff89-5ca1891baf9f/data/data6/current/BP-427817209-172.17.0.2-1733761835089 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T16:31:28,978 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/cluster_cd98e670-d246-b6f3-ff89-5ca1891baf9f/data/data5/current/BP-427817209-172.17.0.2-1733761835089 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T16:31:28,991 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@28e1ba78{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T16:31:28,992 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@702c0733{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T16:31:28,992 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T16:31:28,992 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3264f7d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T16:31:28,993 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e971547{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/hadoop.log.dir/,STOPPED} 2024-12-09T16:31:28,997 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T16:31:28,997 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T16:31:28,997 WARN [BP-427817209-172.17.0.2-1733761835089 heartbeating to localhost/127.0.0.1:39607 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T16:31:28,997 WARN [BP-427817209-172.17.0.2-1733761835089 heartbeating to localhost/127.0.0.1:39607 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-427817209-172.17.0.2-1733761835089 (Datanode Uuid 54b3487c-0a7e-4cdd-b31d-17fa0c40dd20) service to localhost/127.0.0.1:39607 2024-12-09T16:31:28,998 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/cluster_cd98e670-d246-b6f3-ff89-5ca1891baf9f/data/data3/current/BP-427817209-172.17.0.2-1733761835089 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T16:31:28,998 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/cluster_cd98e670-d246-b6f3-ff89-5ca1891baf9f/data/data4/current/BP-427817209-172.17.0.2-1733761835089 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T16:31:29,000 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T16:31:29,023 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4dc44ec6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T16:31:29,024 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@17f6e478{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T16:31:29,024 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T16:31:29,024 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3d33512e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T16:31:29,025 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b895d95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/hadoop.log.dir/,STOPPED} 2024-12-09T16:31:29,031 WARN [BP-427817209-172.17.0.2-1733761835089 heartbeating to localhost/127.0.0.1:39607 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T16:31:29,032 WARN [BP-427817209-172.17.0.2-1733761835089 heartbeating to localhost/127.0.0.1:39607 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-427817209-172.17.0.2-1733761835089 (Datanode Uuid f89b2c38-2a6a-48a7-a790-f1f498b2e863) service to localhost/127.0.0.1:39607 2024-12-09T16:31:29,033 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/cluster_cd98e670-d246-b6f3-ff89-5ca1891baf9f/data/data1/current/BP-427817209-172.17.0.2-1733761835089 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T16:31:29,033 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/cluster_cd98e670-d246-b6f3-ff89-5ca1891baf9f/data/data2/current/BP-427817209-172.17.0.2-1733761835089 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T16:31:29,033 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T16:31:29,034 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T16:31:29,034 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T16:31:29,051 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1843526c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T16:31:29,053 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@776c4d96{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T16:31:29,053 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T16:31:29,053 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70596a48{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T16:31:29,054 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6630be9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5c6a4d2b-43d0-b20e-156b-43b2f969cd52/hadoop.log.dir/,STOPPED} 2024-12-09T16:31:29,086 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-09T16:31:29,165 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down