2024-11-20 20:35:11,130 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-20 20:35:11,146 main DEBUG Took 0.013786 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 20:35:11,147 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 20:35:11,147 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 20:35:11,148 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 20:35:11,150 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 20:35:11,158 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 20:35:11,173 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 20:35:11,175 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 20:35:11,176 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 20:35:11,177 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 20:35:11,177 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 20:35:11,178 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 20:35:11,179 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 20:35:11,179 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 20:35:11,180 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 20:35:11,181 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 20:35:11,182 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 20:35:11,182 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 20:35:11,183 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 20:35:11,183 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 20:35:11,184 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 20:35:11,185 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 20:35:11,185 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 20:35:11,186 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 20:35:11,186 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 20:35:11,187 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 20:35:11,187 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 20:35:11,188 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 20:35:11,188 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 20:35:11,189 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 20:35:11,190 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 20:35:11,190 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 20:35:11,192 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 20:35:11,194 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 20:35:11,196 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 20:35:11,197 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 20:35:11,198 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 20:35:11,199 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 20:35:11,211 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 20:35:11,214 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 20:35:11,217 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 20:35:11,217 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 20:35:11,218 main DEBUG createAppenders(={Console}) 2024-11-20 20:35:11,219 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-20 20:35:11,220 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-20 20:35:11,220 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-20 20:35:11,221 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 20:35:11,221 main DEBUG OutputStream closed 2024-11-20 20:35:11,221 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 20:35:11,222 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 20:35:11,222 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-20 20:35:11,293 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 20:35:11,295 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 20:35:11,296 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 20:35:11,297 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 20:35:11,298 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 20:35:11,298 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 20:35:11,298 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 20:35:11,299 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 20:35:11,299 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 20:35:11,300 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 20:35:11,300 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 20:35:11,300 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 20:35:11,301 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 20:35:11,301 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 20:35:11,301 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 20:35:11,302 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 20:35:11,302 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 20:35:11,303 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 20:35:11,305 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 20:35:11,305 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-20 20:35:11,306 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 20:35:11,306 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-20T20:35:11,550 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986 2024-11-20 20:35:11,553 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 20:35:11,553 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T20:35:11,562 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-20T20:35:11,568 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayCompressed timeout: 13 mins 2024-11-20T20:35:11,590 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T20:35:11,632 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T20:35:11,633 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T20:35:11,644 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-20T20:35:11,658 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/cluster_bbcbe4e8-087e-d341-dffc-b7e2617acb01, deleteOnExit=true 2024-11-20T20:35:11,659 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-20T20:35:11,659 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/test.cache.data in system properties and HBase conf 2024-11-20T20:35:11,660 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/hadoop.tmp.dir in system properties and HBase conf 2024-11-20T20:35:11,661 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/hadoop.log.dir in system properties and HBase conf 2024-11-20T20:35:11,661 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-20T20:35:11,662 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-20T20:35:11,662 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-20T20:35:11,751 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T20:35:11,903 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T20:35:11,908 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-20T20:35:11,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-20T20:35:11,910 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-20T20:35:11,910 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T20:35:11,911 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-20T20:35:11,912 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-20T20:35:11,912 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T20:35:11,913 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T20:35:11,914 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-20T20:35:11,915 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/nfs.dump.dir in system properties and HBase conf 2024-11-20T20:35:11,915 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/java.io.tmpdir in system properties and HBase conf 2024-11-20T20:35:11,916 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T20:35:11,917 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-20T20:35:11,917 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-20T20:35:13,069 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-20T20:35:13,142 INFO [Time-limited test {}] log.Log(170): Logging initialized @2712ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-20T20:35:13,211 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T20:35:13,275 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T20:35:13,294 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T20:35:13,295 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T20:35:13,296 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T20:35:13,314 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T20:35:13,318 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6630be9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/hadoop.log.dir/,AVAILABLE} 2024-11-20T20:35:13,319 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70596a48{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T20:35:13,500 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1843526c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/java.io.tmpdir/jetty-localhost-45037-hadoop-hdfs-3_4_1-tests_jar-_-any-9059105884734879034/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T20:35:13,507 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@776c4d96{HTTP/1.1, (http/1.1)}{localhost:45037} 2024-11-20T20:35:13,507 INFO [Time-limited test {}] server.Server(415): Started @3077ms 2024-11-20T20:35:14,259 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T20:35:14,272 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T20:35:14,275 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T20:35:14,275 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T20:35:14,276 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T20:35:14,277 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b895d95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/hadoop.log.dir/,AVAILABLE} 2024-11-20T20:35:14,278 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3d33512e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T20:35:14,406 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4dc44ec6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/java.io.tmpdir/jetty-localhost-35665-hadoop-hdfs-3_4_1-tests_jar-_-any-14787434536748359528/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T20:35:14,407 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@17f6e478{HTTP/1.1, (http/1.1)}{localhost:35665} 2024-11-20T20:35:14,407 INFO [Time-limited test {}] server.Server(415): Started @3977ms 2024-11-20T20:35:14,466 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T20:35:14,694 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T20:35:14,712 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T20:35:14,738 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T20:35:14,738 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T20:35:14,738 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T20:35:14,745 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e971547{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/hadoop.log.dir/,AVAILABLE} 2024-11-20T20:35:14,746 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3264f7d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T20:35:14,892 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@28e1ba78{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/java.io.tmpdir/jetty-localhost-44685-hadoop-hdfs-3_4_1-tests_jar-_-any-6253335967228992964/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T20:35:14,893 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@702c0733{HTTP/1.1, (http/1.1)}{localhost:44685} 2024-11-20T20:35:14,894 INFO [Time-limited test {}] server.Server(415): Started @4464ms 2024-11-20T20:35:14,897 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T20:35:14,987 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T20:35:14,994 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T20:35:15,004 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T20:35:15,004 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T20:35:15,004 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T20:35:15,007 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bf0fdb4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/hadoop.log.dir/,AVAILABLE} 2024-11-20T20:35:15,007 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6853f5bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T20:35:15,125 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4290616c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/java.io.tmpdir/jetty-localhost-38269-hadoop-hdfs-3_4_1-tests_jar-_-any-16611651379262593905/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T20:35:15,126 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5e25d2b6{HTTP/1.1, (http/1.1)}{localhost:38269} 2024-11-20T20:35:15,126 INFO [Time-limited test {}] server.Server(415): Started @4696ms 2024-11-20T20:35:15,129 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T20:35:16,437 WARN [Thread-121 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/cluster_bbcbe4e8-087e-d341-dffc-b7e2617acb01/data/data2/current/BP-1180650432-172.17.0.2-1732134912528/current, will proceed with Du for space computation calculation, 2024-11-20T20:35:16,437 WARN [Thread-120 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/cluster_bbcbe4e8-087e-d341-dffc-b7e2617acb01/data/data1/current/BP-1180650432-172.17.0.2-1732134912528/current, will proceed with Du for space computation calculation, 2024-11-20T20:35:16,469 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T20:35:16,511 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xba45a58315745fc3 with lease ID 0xd2725eff6eb76caf: Processing first storage report for DS-168a228a-4d47-4268-859c-e933fd2e682c from datanode DatanodeRegistration(127.0.0.1:44519, datanodeUuid=6beff4b0-2a83-454d-9cae-0ce8f6e616ce, infoPort=46511, infoSecurePort=0, ipcPort=46129, storageInfo=lv=-57;cid=testClusterID;nsid=1481022012;c=1732134912528) 2024-11-20T20:35:16,512 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xba45a58315745fc3 with lease ID 0xd2725eff6eb76caf: from storage DS-168a228a-4d47-4268-859c-e933fd2e682c node DatanodeRegistration(127.0.0.1:44519, datanodeUuid=6beff4b0-2a83-454d-9cae-0ce8f6e616ce, infoPort=46511, infoSecurePort=0, ipcPort=46129, storageInfo=lv=-57;cid=testClusterID;nsid=1481022012;c=1732134912528), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-20T20:35:16,513 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xba45a58315745fc3 with lease ID 0xd2725eff6eb76caf: Processing first storage report for DS-eec34e68-2788-4ae9-b1be-042277b2eda1 from datanode DatanodeRegistration(127.0.0.1:44519, datanodeUuid=6beff4b0-2a83-454d-9cae-0ce8f6e616ce, infoPort=46511, infoSecurePort=0, ipcPort=46129, storageInfo=lv=-57;cid=testClusterID;nsid=1481022012;c=1732134912528) 2024-11-20T20:35:16,513 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xba45a58315745fc3 with lease ID 0xd2725eff6eb76caf: from storage DS-eec34e68-2788-4ae9-b1be-042277b2eda1 node DatanodeRegistration(127.0.0.1:44519, datanodeUuid=6beff4b0-2a83-454d-9cae-0ce8f6e616ce, infoPort=46511, infoSecurePort=0, ipcPort=46129, storageInfo=lv=-57;cid=testClusterID;nsid=1481022012;c=1732134912528), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T20:35:16,787 WARN [Thread-131 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/cluster_bbcbe4e8-087e-d341-dffc-b7e2617acb01/data/data3/current/BP-1180650432-172.17.0.2-1732134912528/current, will proceed with Du for space computation calculation, 2024-11-20T20:35:16,787 WARN [Thread-132 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/cluster_bbcbe4e8-087e-d341-dffc-b7e2617acb01/data/data4/current/BP-1180650432-172.17.0.2-1732134912528/current, will proceed with Du for space computation calculation, 2024-11-20T20:35:16,809 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T20:35:16,813 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3eaae0aa304d3071 with lease ID 0xd2725eff6eb76cb0: Processing first storage report for DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a from datanode DatanodeRegistration(127.0.0.1:35033, datanodeUuid=9bc160bb-b0dc-4c50-97e0-2f3d53833c72, infoPort=38233, infoSecurePort=0, ipcPort=46509, storageInfo=lv=-57;cid=testClusterID;nsid=1481022012;c=1732134912528) 2024-11-20T20:35:16,814 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3eaae0aa304d3071 with lease ID 0xd2725eff6eb76cb0: from storage DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a node DatanodeRegistration(127.0.0.1:35033, datanodeUuid=9bc160bb-b0dc-4c50-97e0-2f3d53833c72, infoPort=38233, infoSecurePort=0, ipcPort=46509, storageInfo=lv=-57;cid=testClusterID;nsid=1481022012;c=1732134912528), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T20:35:16,814 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3eaae0aa304d3071 with lease ID 0xd2725eff6eb76cb0: Processing first storage report for DS-8a3deee3-aa03-45d7-8416-0ef990b8a5aa from datanode DatanodeRegistration(127.0.0.1:35033, datanodeUuid=9bc160bb-b0dc-4c50-97e0-2f3d53833c72, infoPort=38233, infoSecurePort=0, ipcPort=46509, storageInfo=lv=-57;cid=testClusterID;nsid=1481022012;c=1732134912528) 2024-11-20T20:35:16,814 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3eaae0aa304d3071 with lease ID 0xd2725eff6eb76cb0: from storage DS-8a3deee3-aa03-45d7-8416-0ef990b8a5aa node DatanodeRegistration(127.0.0.1:35033, datanodeUuid=9bc160bb-b0dc-4c50-97e0-2f3d53833c72, infoPort=38233, infoSecurePort=0, ipcPort=46509, storageInfo=lv=-57;cid=testClusterID;nsid=1481022012;c=1732134912528), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T20:35:16,989 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/cluster_bbcbe4e8-087e-d341-dffc-b7e2617acb01/data/data6/current/BP-1180650432-172.17.0.2-1732134912528/current, will proceed with Du for space computation calculation, 2024-11-20T20:35:16,989 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/cluster_bbcbe4e8-087e-d341-dffc-b7e2617acb01/data/data5/current/BP-1180650432-172.17.0.2-1732134912528/current, will proceed with Du for space computation calculation, 2024-11-20T20:35:17,005 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T20:35:17,009 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7087a926b3401ee1 with lease ID 0xd2725eff6eb76cb1: Processing first storage report for DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c from datanode DatanodeRegistration(127.0.0.1:40323, datanodeUuid=16edd396-61c9-42d4-ae15-5798a67a52d9, infoPort=39759, infoSecurePort=0, ipcPort=42375, storageInfo=lv=-57;cid=testClusterID;nsid=1481022012;c=1732134912528) 2024-11-20T20:35:17,009 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7087a926b3401ee1 with lease ID 0xd2725eff6eb76cb1: from storage DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c node DatanodeRegistration(127.0.0.1:40323, datanodeUuid=16edd396-61c9-42d4-ae15-5798a67a52d9, infoPort=39759, infoSecurePort=0, ipcPort=42375, storageInfo=lv=-57;cid=testClusterID;nsid=1481022012;c=1732134912528), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T20:35:17,010 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7087a926b3401ee1 with lease ID 0xd2725eff6eb76cb1: Processing first storage report for DS-cc68257b-841a-4dc8-b570-940bb0a41a38 from datanode DatanodeRegistration(127.0.0.1:40323, datanodeUuid=16edd396-61c9-42d4-ae15-5798a67a52d9, infoPort=39759, infoSecurePort=0, ipcPort=42375, storageInfo=lv=-57;cid=testClusterID;nsid=1481022012;c=1732134912528) 2024-11-20T20:35:17,010 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7087a926b3401ee1 with lease ID 0xd2725eff6eb76cb1: from storage DS-cc68257b-841a-4dc8-b570-940bb0a41a38 node DatanodeRegistration(127.0.0.1:40323, datanodeUuid=16edd396-61c9-42d4-ae15-5798a67a52d9, infoPort=39759, infoSecurePort=0, ipcPort=42375, storageInfo=lv=-57;cid=testClusterID;nsid=1481022012;c=1732134912528), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T20:35:17,093 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986 2024-11-20T20:35:17,166 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/cluster_bbcbe4e8-087e-d341-dffc-b7e2617acb01/zookeeper_0, clientPort=64093, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/cluster_bbcbe4e8-087e-d341-dffc-b7e2617acb01/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/cluster_bbcbe4e8-087e-d341-dffc-b7e2617acb01/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T20:35:17,177 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64093 2024-11-20T20:35:17,204 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T20:35:17,208 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T20:35:17,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741825_1001 (size=7) 2024-11-20T20:35:17,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741825_1001 (size=7) 2024-11-20T20:35:17,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741825_1001 (size=7) 2024-11-20T20:35:17,918 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6 with version=8 2024-11-20T20:35:17,919 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/hbase-staging 2024-11-20T20:35:18,218 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d7d59561a64e:0 server-side Connection retries=45 2024-11-20T20:35:18,228 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T20:35:18,228 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T20:35:18,233 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T20:35:18,233 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T20:35:18,233 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T20:35:18,363 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T20:35:18,419 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T20:35:18,428 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T20:35:18,432 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T20:35:18,460 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 76135 (auto-detected) 2024-11-20T20:35:18,462 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-20T20:35:18,487 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46641 2024-11-20T20:35:18,511 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46641 connecting to ZooKeeper ensemble=127.0.0.1:64093 2024-11-20T20:35:18,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:466410x0, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T20:35:18,616 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46641-0x10159f54fc60000 connected 2024-11-20T20:35:18,890 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T20:35:18,894 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T20:35:18,907 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T20:35:18,911 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6, hbase.cluster.distributed=false 2024-11-20T20:35:18,941 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T20:35:18,946 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46641 2024-11-20T20:35:18,947 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46641 2024-11-20T20:35:18,948 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46641 2024-11-20T20:35:18,949 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46641 2024-11-20T20:35:18,950 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46641 2024-11-20T20:35:19,066 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d7d59561a64e:0 server-side Connection retries=45 2024-11-20T20:35:19,068 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T20:35:19,068 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T20:35:19,069 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T20:35:19,069 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T20:35:19,069 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T20:35:19,072 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T20:35:19,074 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T20:35:19,075 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36639 2024-11-20T20:35:19,077 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36639 connecting to ZooKeeper ensemble=127.0.0.1:64093 2024-11-20T20:35:19,078 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T20:35:19,083 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T20:35:19,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:366390x0, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T20:35:19,217 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36639-0x10159f54fc60001 connected 2024-11-20T20:35:19,217 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36639-0x10159f54fc60001, quorum=127.0.0.1:64093, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T20:35:19,221 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T20:35:19,229 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T20:35:19,232 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36639-0x10159f54fc60001, quorum=127.0.0.1:64093, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T20:35:19,238 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36639-0x10159f54fc60001, quorum=127.0.0.1:64093, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T20:35:19,240 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36639 2024-11-20T20:35:19,241 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36639 2024-11-20T20:35:19,244 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36639 2024-11-20T20:35:19,245 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36639 2024-11-20T20:35:19,245 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36639 2024-11-20T20:35:19,262 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d7d59561a64e:0 server-side Connection retries=45 2024-11-20T20:35:19,263 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T20:35:19,263 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T20:35:19,263 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T20:35:19,263 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T20:35:19,264 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T20:35:19,264 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T20:35:19,264 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T20:35:19,269 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33553 2024-11-20T20:35:19,272 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33553 connecting to ZooKeeper ensemble=127.0.0.1:64093 2024-11-20T20:35:19,273 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T20:35:19,279 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T20:35:19,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:335530x0, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T20:35:19,356 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:335530x0, quorum=127.0.0.1:64093, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T20:35:19,357 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33553-0x10159f54fc60002 connected 2024-11-20T20:35:19,357 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T20:35:19,359 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T20:35:19,360 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33553-0x10159f54fc60002, quorum=127.0.0.1:64093, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T20:35:19,362 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33553-0x10159f54fc60002, quorum=127.0.0.1:64093, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T20:35:19,364 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33553 2024-11-20T20:35:19,364 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33553 2024-11-20T20:35:19,365 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33553 2024-11-20T20:35:19,366 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33553 2024-11-20T20:35:19,366 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33553 2024-11-20T20:35:19,382 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d7d59561a64e:0 server-side Connection retries=45 2024-11-20T20:35:19,382 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T20:35:19,382 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T20:35:19,382 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T20:35:19,383 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T20:35:19,383 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T20:35:19,383 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T20:35:19,383 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T20:35:19,384 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36243 2024-11-20T20:35:19,386 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36243 connecting to ZooKeeper ensemble=127.0.0.1:64093 2024-11-20T20:35:19,388 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T20:35:19,391 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T20:35:19,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:362430x0, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T20:35:19,411 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36243-0x10159f54fc60003 connected 2024-11-20T20:35:19,411 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36243-0x10159f54fc60003, quorum=127.0.0.1:64093, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T20:35:19,412 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T20:35:19,414 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T20:35:19,415 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36243-0x10159f54fc60003, quorum=127.0.0.1:64093, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T20:35:19,417 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36243-0x10159f54fc60003, quorum=127.0.0.1:64093, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T20:35:19,420 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36243 2024-11-20T20:35:19,421 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36243 2024-11-20T20:35:19,421 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36243 2024-11-20T20:35:19,422 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36243 2024-11-20T20:35:19,422 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36243 2024-11-20T20:35:19,438 DEBUG [M:0;d7d59561a64e:46641 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d7d59561a64e:46641 2024-11-20T20:35:19,439 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d7d59561a64e,46641,1732134918030 2024-11-20T20:35:19,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36243-0x10159f54fc60003, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T20:35:19,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36639-0x10159f54fc60001, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T20:35:19,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33553-0x10159f54fc60002, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T20:35:19,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T20:35:19,560 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d7d59561a64e,46641,1732134918030 2024-11-20T20:35:19,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33553-0x10159f54fc60002, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T20:35:19,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33553-0x10159f54fc60002, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:35:19,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36243-0x10159f54fc60003, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T20:35:19,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:35:19,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36243-0x10159f54fc60003, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:35:19,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36639-0x10159f54fc60001, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T20:35:19,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36639-0x10159f54fc60001, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:35:19,614 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T20:35:19,615 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d7d59561a64e,46641,1732134918030 from backup master directory 2024-11-20T20:35:19,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d7d59561a64e,46641,1732134918030 2024-11-20T20:35:19,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36639-0x10159f54fc60001, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T20:35:19,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36243-0x10159f54fc60003, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T20:35:19,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T20:35:19,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33553-0x10159f54fc60002, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T20:35:19,634 WARN [master/d7d59561a64e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T20:35:19,634 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d7d59561a64e,46641,1732134918030 2024-11-20T20:35:19,637 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T20:35:19,639 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T20:35:19,750 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/hbase.id] with ID: 92b6d327-3593-434a-bf0b-fee82840fcda 2024-11-20T20:35:19,751 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/.tmp/hbase.id 2024-11-20T20:35:19,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741826_1002 (size=42) 2024-11-20T20:35:19,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741826_1002 (size=42) 2024-11-20T20:35:19,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741826_1002 (size=42) 2024-11-20T20:35:19,804 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/.tmp/hbase.id]:[hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/hbase.id] 2024-11-20T20:35:19,888 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T20:35:19,897 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T20:35:19,933 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 31ms. 2024-11-20T20:35:19,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33553-0x10159f54fc60002, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:35:19,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36639-0x10159f54fc60001, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:35:19,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:35:19,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36243-0x10159f54fc60003, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:35:20,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741827_1003 (size=196) 2024-11-20T20:35:20,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741827_1003 (size=196) 2024-11-20T20:35:20,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741827_1003 (size=196) 2024-11-20T20:35:20,068 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T20:35:20,071 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T20:35:20,087 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:20,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741828_1004 (size=1189) 2024-11-20T20:35:20,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741828_1004 (size=1189) 2024-11-20T20:35:20,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741828_1004 (size=1189) 2024-11-20T20:35:20,647 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/data/master/store 2024-11-20T20:35:20,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741829_1005 (size=34) 2024-11-20T20:35:20,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741829_1005 (size=34) 2024-11-20T20:35:20,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741829_1005 (size=34) 2024-11-20T20:35:20,731 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T20:35:20,735 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:20,736 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T20:35:20,737 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T20:35:20,737 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T20:35:20,738 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T20:35:20,739 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T20:35:20,739 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T20:35:20,740 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732134920736Disabling compacts and flushes for region at 1732134920736Disabling writes for close at 1732134920739 (+3 ms)Writing region close event to WAL at 1732134920739Closed at 1732134920739 2024-11-20T20:35:20,743 WARN [master/d7d59561a64e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/data/master/store/.initializing 2024-11-20T20:35:20,743 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/WALs/d7d59561a64e,46641,1732134918030 2024-11-20T20:35:20,751 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T20:35:20,767 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d7d59561a64e%2C46641%2C1732134918030, suffix=, logDir=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/WALs/d7d59561a64e,46641,1732134918030, archiveDir=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/oldWALs, maxLogs=10 2024-11-20T20:35:20,804 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/WALs/d7d59561a64e,46641,1732134918030/d7d59561a64e%2C46641%2C1732134918030.1732134920771, exclude list is [], retry=0 2024-11-20T20:35:20,828 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:35:20,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:20,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:20,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:20,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-20T20:35:20,883 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/WALs/d7d59561a64e,46641,1732134918030/d7d59561a64e%2C46641%2C1732134918030.1732134920771 2024-11-20T20:35:20,884 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:39759:39759),(127.0.0.1/127.0.0.1:38233:38233)] 2024-11-20T20:35:20,885 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:35:20,885 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:20,889 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T20:35:20,890 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T20:35:20,937 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T20:35:20,962 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T20:35:20,966 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:20,969 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T20:35:20,970 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T20:35:20,974 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T20:35:20,975 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:20,976 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:20,977 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T20:35:20,981 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T20:35:20,981 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:20,982 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:20,983 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T20:35:20,986 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T20:35:20,986 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:20,988 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:20,988 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T20:35:20,993 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T20:35:20,995 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T20:35:21,002 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T20:35:21,002 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T20:35:21,007 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T20:35:21,011 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T20:35:21,016 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T20:35:21,017 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62537598, jitterRate=-0.0681171715259552}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T20:35:21,026 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732134920903Initializing all the Stores at 1732134920905 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732134920906 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134920906Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134920907 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134920907Cleaning up temporary data from old regions at 1732134921002 (+95 ms)Region opened successfully at 1732134921025 (+23 ms) 2024-11-20T20:35:21,027 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T20:35:21,075 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@184c4dcf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d7d59561a64e/172.17.0.2:0 2024-11-20T20:35:21,104 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T20:35:21,114 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T20:35:21,114 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T20:35:21,117 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T20:35:21,118 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-20T20:35:21,123 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-20T20:35:21,123 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T20:35:21,148 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T20:35:21,156 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T20:35:21,283 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T20:35:21,292 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T20:35:21,295 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T20:35:21,315 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T20:35:21,317 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T20:35:21,321 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T20:35:21,334 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T20:35:21,336 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T20:35:21,346 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T20:35:21,368 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T20:35:21,378 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T20:35:21,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T20:35:21,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36243-0x10159f54fc60003, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T20:35:21,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36639-0x10159f54fc60001, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T20:35:21,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33553-0x10159f54fc60002, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T20:35:21,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33553-0x10159f54fc60002, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:35:21,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:35:21,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36639-0x10159f54fc60001, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:35:21,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36243-0x10159f54fc60003, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:35:21,393 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d7d59561a64e,46641,1732134918030, sessionid=0x10159f54fc60000, setting cluster-up flag (Was=false) 2024-11-20T20:35:21,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36639-0x10159f54fc60001, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:35:21,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36243-0x10159f54fc60003, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:35:21,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33553-0x10159f54fc60002, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:35:21,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:35:21,462 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T20:35:21,466 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d7d59561a64e,46641,1732134918030 2024-11-20T20:35:21,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36639-0x10159f54fc60001, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:35:21,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33553-0x10159f54fc60002, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:35:21,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:35:21,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36243-0x10159f54fc60003, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:35:21,536 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T20:35:21,539 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d7d59561a64e,46641,1732134918030 2024-11-20T20:35:21,550 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T20:35:21,633 INFO [RS:2;d7d59561a64e:36243 {}] regionserver.HRegionServer(746): ClusterId : 92b6d327-3593-434a-bf0b-fee82840fcda 2024-11-20T20:35:21,633 INFO [RS:0;d7d59561a64e:36639 {}] regionserver.HRegionServer(746): ClusterId : 92b6d327-3593-434a-bf0b-fee82840fcda 2024-11-20T20:35:21,633 INFO [RS:1;d7d59561a64e:33553 {}] regionserver.HRegionServer(746): ClusterId : 92b6d327-3593-434a-bf0b-fee82840fcda 2024-11-20T20:35:21,636 DEBUG [RS:2;d7d59561a64e:36243 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T20:35:21,636 DEBUG [RS:1;d7d59561a64e:33553 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T20:35:21,636 DEBUG [RS:0;d7d59561a64e:36639 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T20:35:21,651 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T20:35:21,664 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T20:35:21,665 DEBUG [RS:2;d7d59561a64e:36243 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T20:35:21,665 DEBUG [RS:1;d7d59561a64e:33553 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T20:35:21,665 DEBUG [RS:0;d7d59561a64e:36639 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T20:35:21,665 DEBUG [RS:2;d7d59561a64e:36243 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T20:35:21,665 DEBUG [RS:1;d7d59561a64e:33553 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T20:35:21,665 DEBUG [RS:0;d7d59561a64e:36639 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T20:35:21,673 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T20:35:21,681 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d7d59561a64e,46641,1732134918030 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T20:35:21,767 DEBUG [RS:0;d7d59561a64e:36639 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T20:35:21,767 DEBUG [RS:1;d7d59561a64e:33553 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T20:35:21,768 DEBUG [RS:0;d7d59561a64e:36639 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1217c828, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d7d59561a64e/172.17.0.2:0 2024-11-20T20:35:21,768 DEBUG [RS:2;d7d59561a64e:36243 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T20:35:21,769 DEBUG [RS:2;d7d59561a64e:36243 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25212925, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d7d59561a64e/172.17.0.2:0 2024-11-20T20:35:21,770 DEBUG [RS:1;d7d59561a64e:33553 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bb7ae5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d7d59561a64e/172.17.0.2:0 2024-11-20T20:35:21,771 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d7d59561a64e:0, corePoolSize=5, maxPoolSize=5 2024-11-20T20:35:21,771 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d7d59561a64e:0, corePoolSize=5, maxPoolSize=5 2024-11-20T20:35:21,772 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d7d59561a64e:0, corePoolSize=5, maxPoolSize=5 2024-11-20T20:35:21,772 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d7d59561a64e:0, corePoolSize=5, maxPoolSize=5 2024-11-20T20:35:21,772 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d7d59561a64e:0, corePoolSize=10, maxPoolSize=10 2024-11-20T20:35:21,772 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:21,773 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d7d59561a64e:0, corePoolSize=2, maxPoolSize=2 2024-11-20T20:35:21,773 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:21,797 DEBUG [RS:0;d7d59561a64e:36639 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d7d59561a64e:36639 2024-11-20T20:35:21,801 INFO [RS:0;d7d59561a64e:36639 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T20:35:21,801 INFO [RS:0;d7d59561a64e:36639 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T20:35:21,801 DEBUG [RS:0;d7d59561a64e:36639 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T20:35:21,804 DEBUG [RS:2;d7d59561a64e:36243 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;d7d59561a64e:36243 2024-11-20T20:35:21,805 INFO [RS:0;d7d59561a64e:36639 {}] regionserver.HRegionServer(2659): reportForDuty to master=d7d59561a64e,46641,1732134918030 with port=36639, startcode=1732134919021 2024-11-20T20:35:21,806 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T20:35:21,806 DEBUG [RS:1;d7d59561a64e:33553 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;d7d59561a64e:33553 2024-11-20T20:35:21,806 INFO [RS:2;d7d59561a64e:36243 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T20:35:21,807 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T20:35:21,807 INFO [RS:1;d7d59561a64e:33553 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T20:35:21,807 INFO [RS:2;d7d59561a64e:36243 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T20:35:21,807 INFO [RS:1;d7d59561a64e:33553 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T20:35:21,807 DEBUG [RS:2;d7d59561a64e:36243 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T20:35:21,807 DEBUG [RS:1;d7d59561a64e:33553 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T20:35:21,808 INFO [RS:1;d7d59561a64e:33553 {}] regionserver.HRegionServer(2659): reportForDuty to master=d7d59561a64e,46641,1732134918030 with port=33553, startcode=1732134919262 2024-11-20T20:35:21,810 INFO [RS:2;d7d59561a64e:36243 {}] regionserver.HRegionServer(2659): reportForDuty to master=d7d59561a64e,46641,1732134918030 with port=36243, startcode=1732134919381 2024-11-20T20:35:21,815 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:21,815 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T20:35:21,817 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732134951817 2024-11-20T20:35:21,819 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T20:35:21,821 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T20:35:21,822 DEBUG [RS:0;d7d59561a64e:36639 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T20:35:21,824 DEBUG [RS:2;d7d59561a64e:36243 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T20:35:21,826 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T20:35:21,826 DEBUG [RS:1;d7d59561a64e:33553 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T20:35:21,828 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T20:35:21,829 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T20:35:21,829 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T20:35:21,834 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:21,846 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T20:35:21,848 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T20:35:21,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741831_1007 (size=1321) 2024-11-20T20:35:21,850 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T20:35:21,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741831_1007 (size=1321) 2024-11-20T20:35:21,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741831_1007 (size=1321) 2024-11-20T20:35:21,852 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T20:35:21,852 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T20:35:21,855 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d7d59561a64e:0:becomeActiveMaster-HFileCleaner.large.0-1732134921853,5,FailOnTimeoutGroup] 2024-11-20T20:35:21,856 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d7d59561a64e:0:becomeActiveMaster-HFileCleaner.small.0-1732134921855,5,FailOnTimeoutGroup] 2024-11-20T20:35:21,856 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:21,857 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T20:35:21,858 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:21,858 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:21,876 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56561, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T20:35:21,876 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50803, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T20:35:21,876 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36595, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T20:35:21,885 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46641 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d7d59561a64e,33553,1732134919262 2024-11-20T20:35:21,889 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46641 {}] master.ServerManager(517): Registering regionserver=d7d59561a64e,33553,1732134919262 2024-11-20T20:35:21,905 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46641 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d7d59561a64e,36243,1732134919381 2024-11-20T20:35:21,905 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46641 {}] master.ServerManager(517): Registering regionserver=d7d59561a64e,36243,1732134919381 2024-11-20T20:35:21,911 DEBUG [RS:1;d7d59561a64e:33553 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6 2024-11-20T20:35:21,911 DEBUG [RS:1;d7d59561a64e:33553 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33605 2024-11-20T20:35:21,911 DEBUG [RS:1;d7d59561a64e:33553 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T20:35:21,913 DEBUG [RS:2;d7d59561a64e:36243 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6 2024-11-20T20:35:21,913 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46641 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d7d59561a64e,36639,1732134919021 2024-11-20T20:35:21,913 DEBUG [RS:2;d7d59561a64e:36243 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33605 2024-11-20T20:35:21,913 DEBUG [RS:2;d7d59561a64e:36243 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T20:35:21,914 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46641 {}] master.ServerManager(517): Registering regionserver=d7d59561a64e,36639,1732134919021 2024-11-20T20:35:21,918 DEBUG [RS:0;d7d59561a64e:36639 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6 2024-11-20T20:35:21,919 DEBUG [RS:0;d7d59561a64e:36639 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33605 2024-11-20T20:35:21,919 DEBUG [RS:0;d7d59561a64e:36639 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T20:35:21,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T20:35:22,015 DEBUG [RS:2;d7d59561a64e:36243 {}] zookeeper.ZKUtil(111): regionserver:36243-0x10159f54fc60003, quorum=127.0.0.1:64093, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d7d59561a64e,36243,1732134919381 2024-11-20T20:35:22,016 WARN [RS:2;d7d59561a64e:36243 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T20:35:22,016 INFO [RS:2;d7d59561a64e:36243 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:22,016 DEBUG [RS:2;d7d59561a64e:36243 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36243,1732134919381 2024-11-20T20:35:22,018 DEBUG [RS:0;d7d59561a64e:36639 {}] zookeeper.ZKUtil(111): regionserver:36639-0x10159f54fc60001, quorum=127.0.0.1:64093, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d7d59561a64e,36639,1732134919021 2024-11-20T20:35:22,018 WARN [RS:0;d7d59561a64e:36639 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T20:35:22,019 INFO [RS:0;d7d59561a64e:36639 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:22,019 DEBUG [RS:0;d7d59561a64e:36639 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021 2024-11-20T20:35:22,020 DEBUG [RS:1;d7d59561a64e:33553 {}] zookeeper.ZKUtil(111): regionserver:33553-0x10159f54fc60002, quorum=127.0.0.1:64093, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d7d59561a64e,33553,1732134919262 2024-11-20T20:35:22,020 WARN [RS:1;d7d59561a64e:33553 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T20:35:22,021 INFO [RS:1;d7d59561a64e:33553 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:22,021 DEBUG [RS:1;d7d59561a64e:33553 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,33553,1732134919262 2024-11-20T20:35:22,021 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d7d59561a64e,36243,1732134919381] 2024-11-20T20:35:22,021 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d7d59561a64e,36639,1732134919021] 2024-11-20T20:35:22,021 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d7d59561a64e,33553,1732134919262] 2024-11-20T20:35:22,072 INFO [RS:2;d7d59561a64e:36243 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T20:35:22,072 INFO [RS:1;d7d59561a64e:33553 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T20:35:22,076 INFO [RS:0;d7d59561a64e:36639 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T20:35:22,098 INFO [RS:2;d7d59561a64e:36243 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T20:35:22,100 INFO [RS:1;d7d59561a64e:33553 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T20:35:22,108 INFO [RS:0;d7d59561a64e:36639 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T20:35:22,109 INFO [RS:2;d7d59561a64e:36243 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T20:35:22,109 INFO [RS:1;d7d59561a64e:33553 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T20:35:22,109 INFO [RS:2;d7d59561a64e:36243 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,109 INFO [RS:1;d7d59561a64e:33553 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,110 INFO [RS:0;d7d59561a64e:36639 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T20:35:22,110 INFO [RS:0;d7d59561a64e:36639 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,116 INFO [RS:2;d7d59561a64e:36243 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T20:35:22,116 INFO [RS:1;d7d59561a64e:33553 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T20:35:22,117 INFO [RS:0;d7d59561a64e:36639 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T20:35:22,125 INFO [RS:2;d7d59561a64e:36243 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T20:35:22,127 INFO [RS:2;d7d59561a64e:36243 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,128 DEBUG [RS:2;d7d59561a64e:36243 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,128 DEBUG [RS:2;d7d59561a64e:36243 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,128 DEBUG [RS:2;d7d59561a64e:36243 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,128 DEBUG [RS:2;d7d59561a64e:36243 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,128 DEBUG [RS:2;d7d59561a64e:36243 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,128 DEBUG [RS:2;d7d59561a64e:36243 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0, corePoolSize=2, maxPoolSize=2 2024-11-20T20:35:22,129 DEBUG [RS:2;d7d59561a64e:36243 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,129 DEBUG [RS:2;d7d59561a64e:36243 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,129 DEBUG [RS:2;d7d59561a64e:36243 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,129 DEBUG [RS:2;d7d59561a64e:36243 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,129 DEBUG [RS:2;d7d59561a64e:36243 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,129 DEBUG [RS:2;d7d59561a64e:36243 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,130 DEBUG [RS:2;d7d59561a64e:36243 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d7d59561a64e:0, corePoolSize=3, maxPoolSize=3 2024-11-20T20:35:22,130 DEBUG [RS:2;d7d59561a64e:36243 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d7d59561a64e:0, corePoolSize=3, maxPoolSize=3 2024-11-20T20:35:22,130 INFO [RS:1;d7d59561a64e:33553 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T20:35:22,130 INFO [RS:1;d7d59561a64e:33553 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,131 DEBUG [RS:1;d7d59561a64e:33553 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,131 INFO [RS:0;d7d59561a64e:36639 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T20:35:22,131 INFO [RS:0;d7d59561a64e:36639 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,131 DEBUG [RS:0;d7d59561a64e:36639 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,132 DEBUG [RS:1;d7d59561a64e:33553 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,132 DEBUG [RS:1;d7d59561a64e:33553 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,132 DEBUG [RS:0;d7d59561a64e:36639 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,132 DEBUG [RS:1;d7d59561a64e:33553 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,132 DEBUG [RS:0;d7d59561a64e:36639 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,132 DEBUG [RS:1;d7d59561a64e:33553 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,132 DEBUG [RS:0;d7d59561a64e:36639 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,132 DEBUG [RS:1;d7d59561a64e:33553 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0, corePoolSize=2, maxPoolSize=2 2024-11-20T20:35:22,132 DEBUG [RS:0;d7d59561a64e:36639 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,132 DEBUG [RS:1;d7d59561a64e:33553 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,132 DEBUG [RS:0;d7d59561a64e:36639 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0, corePoolSize=2, maxPoolSize=2 2024-11-20T20:35:22,132 DEBUG [RS:1;d7d59561a64e:33553 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,132 DEBUG [RS:0;d7d59561a64e:36639 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,133 DEBUG [RS:1;d7d59561a64e:33553 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,133 DEBUG [RS:0;d7d59561a64e:36639 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,133 DEBUG [RS:1;d7d59561a64e:33553 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,133 DEBUG [RS:0;d7d59561a64e:36639 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,133 DEBUG [RS:1;d7d59561a64e:33553 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,133 DEBUG [RS:0;d7d59561a64e:36639 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,133 DEBUG [RS:1;d7d59561a64e:33553 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,133 DEBUG [RS:0;d7d59561a64e:36639 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,133 DEBUG [RS:0;d7d59561a64e:36639 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d7d59561a64e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T20:35:22,133 DEBUG [RS:0;d7d59561a64e:36639 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d7d59561a64e:0, corePoolSize=3, maxPoolSize=3 2024-11-20T20:35:22,134 DEBUG [RS:0;d7d59561a64e:36639 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d7d59561a64e:0, corePoolSize=3, maxPoolSize=3 2024-11-20T20:35:22,134 DEBUG [RS:1;d7d59561a64e:33553 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d7d59561a64e:0, corePoolSize=3, maxPoolSize=3 2024-11-20T20:35:22,134 DEBUG [RS:1;d7d59561a64e:33553 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d7d59561a64e:0, corePoolSize=3, maxPoolSize=3 2024-11-20T20:35:22,152 INFO [RS:0;d7d59561a64e:36639 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,152 INFO [RS:2;d7d59561a64e:36243 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,152 INFO [RS:0;d7d59561a64e:36639 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,152 INFO [RS:2;d7d59561a64e:36243 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,152 INFO [RS:2;d7d59561a64e:36243 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,152 INFO [RS:0;d7d59561a64e:36639 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,153 INFO [RS:0;d7d59561a64e:36639 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,153 INFO [RS:2;d7d59561a64e:36243 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,153 INFO [RS:2;d7d59561a64e:36243 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,153 INFO [RS:0;d7d59561a64e:36639 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,153 INFO [RS:2;d7d59561a64e:36243 {}] hbase.ChoreService(168): Chore ScheduledChore name=d7d59561a64e,36243,1732134919381-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T20:35:22,153 INFO [RS:0;d7d59561a64e:36639 {}] hbase.ChoreService(168): Chore ScheduledChore name=d7d59561a64e,36639,1732134919021-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T20:35:22,167 INFO [RS:1;d7d59561a64e:33553 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,167 INFO [RS:1;d7d59561a64e:33553 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,167 INFO [RS:1;d7d59561a64e:33553 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,167 INFO [RS:1;d7d59561a64e:33553 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,167 INFO [RS:1;d7d59561a64e:33553 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,167 INFO [RS:1;d7d59561a64e:33553 {}] hbase.ChoreService(168): Chore ScheduledChore name=d7d59561a64e,33553,1732134919262-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T20:35:22,192 INFO [RS:0;d7d59561a64e:36639 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T20:35:22,194 INFO [RS:2;d7d59561a64e:36243 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T20:35:22,195 INFO [RS:2;d7d59561a64e:36243 {}] hbase.ChoreService(168): Chore ScheduledChore name=d7d59561a64e,36243,1732134919381-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,196 INFO [RS:2;d7d59561a64e:36243 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,196 INFO [RS:2;d7d59561a64e:36243 {}] regionserver.Replication(171): d7d59561a64e,36243,1732134919381 started 2024-11-20T20:35:22,200 INFO [RS:0;d7d59561a64e:36639 {}] hbase.ChoreService(168): Chore ScheduledChore name=d7d59561a64e,36639,1732134919021-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,200 INFO [RS:0;d7d59561a64e:36639 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,200 INFO [RS:0;d7d59561a64e:36639 {}] regionserver.Replication(171): d7d59561a64e,36639,1732134919021 started 2024-11-20T20:35:22,201 INFO [RS:1;d7d59561a64e:33553 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T20:35:22,201 INFO [RS:1;d7d59561a64e:33553 {}] hbase.ChoreService(168): Chore ScheduledChore name=d7d59561a64e,33553,1732134919262-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,201 INFO [RS:1;d7d59561a64e:33553 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,201 INFO [RS:1;d7d59561a64e:33553 {}] regionserver.Replication(171): d7d59561a64e,33553,1732134919262 started 2024-11-20T20:35:22,224 INFO [RS:2;d7d59561a64e:36243 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,225 INFO [RS:2;d7d59561a64e:36243 {}] regionserver.HRegionServer(1482): Serving as d7d59561a64e,36243,1732134919381, RpcServer on d7d59561a64e/172.17.0.2:36243, sessionid=0x10159f54fc60003 2024-11-20T20:35:22,226 INFO [RS:1;d7d59561a64e:33553 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,226 DEBUG [RS:2;d7d59561a64e:36243 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T20:35:22,227 INFO [RS:1;d7d59561a64e:33553 {}] regionserver.HRegionServer(1482): Serving as d7d59561a64e,33553,1732134919262, RpcServer on d7d59561a64e/172.17.0.2:33553, sessionid=0x10159f54fc60002 2024-11-20T20:35:22,227 DEBUG [RS:2;d7d59561a64e:36243 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d7d59561a64e,36243,1732134919381 2024-11-20T20:35:22,227 DEBUG [RS:1;d7d59561a64e:33553 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T20:35:22,227 DEBUG [RS:1;d7d59561a64e:33553 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d7d59561a64e,33553,1732134919262 2024-11-20T20:35:22,227 DEBUG [RS:1;d7d59561a64e:33553 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd7d59561a64e,33553,1732134919262' 2024-11-20T20:35:22,228 DEBUG [RS:1;d7d59561a64e:33553 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T20:35:22,229 DEBUG [RS:2;d7d59561a64e:36243 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd7d59561a64e,36243,1732134919381' 2024-11-20T20:35:22,229 DEBUG [RS:2;d7d59561a64e:36243 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T20:35:22,230 DEBUG [RS:1;d7d59561a64e:33553 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T20:35:22,230 INFO [RS:0;d7d59561a64e:36639 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:22,230 INFO [RS:0;d7d59561a64e:36639 {}] regionserver.HRegionServer(1482): Serving as d7d59561a64e,36639,1732134919021, RpcServer on d7d59561a64e/172.17.0.2:36639, sessionid=0x10159f54fc60001 2024-11-20T20:35:22,230 DEBUG [RS:0;d7d59561a64e:36639 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T20:35:22,231 DEBUG [RS:0;d7d59561a64e:36639 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d7d59561a64e,36639,1732134919021 2024-11-20T20:35:22,231 DEBUG [RS:0;d7d59561a64e:36639 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd7d59561a64e,36639,1732134919021' 2024-11-20T20:35:22,231 DEBUG [RS:0;d7d59561a64e:36639 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T20:35:22,231 DEBUG [RS:1;d7d59561a64e:33553 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T20:35:22,231 DEBUG [RS:1;d7d59561a64e:33553 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T20:35:22,231 DEBUG [RS:1;d7d59561a64e:33553 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d7d59561a64e,33553,1732134919262 2024-11-20T20:35:22,231 DEBUG [RS:1;d7d59561a64e:33553 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd7d59561a64e,33553,1732134919262' 2024-11-20T20:35:22,231 DEBUG [RS:1;d7d59561a64e:33553 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T20:35:22,233 DEBUG [RS:0;d7d59561a64e:36639 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T20:35:22,234 DEBUG [RS:2;d7d59561a64e:36243 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T20:35:22,234 DEBUG [RS:1;d7d59561a64e:33553 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T20:35:22,235 DEBUG [RS:2;d7d59561a64e:36243 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T20:35:22,235 DEBUG [RS:2;d7d59561a64e:36243 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T20:35:22,235 DEBUG [RS:2;d7d59561a64e:36243 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d7d59561a64e,36243,1732134919381 2024-11-20T20:35:22,235 DEBUG [RS:0;d7d59561a64e:36639 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T20:35:22,236 DEBUG [RS:2;d7d59561a64e:36243 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd7d59561a64e,36243,1732134919381' 2024-11-20T20:35:22,236 DEBUG [RS:0;d7d59561a64e:36639 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T20:35:22,236 DEBUG [RS:2;d7d59561a64e:36243 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T20:35:22,236 DEBUG [RS:0;d7d59561a64e:36639 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d7d59561a64e,36639,1732134919021 2024-11-20T20:35:22,236 DEBUG [RS:0;d7d59561a64e:36639 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd7d59561a64e,36639,1732134919021' 2024-11-20T20:35:22,236 DEBUG [RS:0;d7d59561a64e:36639 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T20:35:22,236 DEBUG [RS:1;d7d59561a64e:33553 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T20:35:22,236 INFO [RS:1;d7d59561a64e:33553 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T20:35:22,236 INFO [RS:1;d7d59561a64e:33553 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T20:35:22,238 DEBUG [RS:2;d7d59561a64e:36243 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T20:35:22,239 DEBUG [RS:2;d7d59561a64e:36243 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T20:35:22,239 INFO [RS:2;d7d59561a64e:36243 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T20:35:22,239 INFO [RS:2;d7d59561a64e:36243 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T20:35:22,240 DEBUG [RS:0;d7d59561a64e:36639 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T20:35:22,251 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T20:35:22,252 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6 2024-11-20T20:35:22,253 DEBUG [RS:0;d7d59561a64e:36639 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T20:35:22,253 INFO [RS:0;d7d59561a64e:36639 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T20:35:22,253 INFO [RS:0;d7d59561a64e:36639 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T20:35:22,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741832_1008 (size=32) 2024-11-20T20:35:22,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741832_1008 (size=32) 2024-11-20T20:35:22,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741832_1008 (size=32) 2024-11-20T20:35:22,336 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:22,343 INFO [RS:2;d7d59561a64e:36243 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T20:35:22,345 INFO [RS:1;d7d59561a64e:33553 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T20:35:22,353 INFO [RS:2;d7d59561a64e:36243 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d7d59561a64e%2C36243%2C1732134919381, suffix=, logDir=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36243,1732134919381, archiveDir=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/oldWALs, maxLogs=32 2024-11-20T20:35:22,354 INFO [RS:1;d7d59561a64e:33553 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d7d59561a64e%2C33553%2C1732134919262, suffix=, logDir=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,33553,1732134919262, archiveDir=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/oldWALs, maxLogs=32 2024-11-20T20:35:22,356 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T20:35:22,357 INFO [RS:0;d7d59561a64e:36639 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T20:35:22,365 INFO [RS:0;d7d59561a64e:36639 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d7d59561a64e%2C36639%2C1732134919021, suffix=, logDir=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021, archiveDir=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/oldWALs, maxLogs=32 2024-11-20T20:35:22,377 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T20:35:22,377 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:22,384 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T20:35:22,385 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T20:35:22,397 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T20:35:22,397 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:22,404 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T20:35:22,405 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T20:35:22,406 DEBUG [RS:1;d7d59561a64e:33553 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,33553,1732134919262/d7d59561a64e%2C33553%2C1732134919262.1732134922358, exclude list is [], retry=0 2024-11-20T20:35:22,411 DEBUG [RS:0;d7d59561a64e:36639 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021/d7d59561a64e%2C36639%2C1732134919021.1732134922368, exclude list is [], retry=0 2024-11-20T20:35:22,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:22,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:22,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:22,417 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T20:35:22,417 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:22,418 DEBUG [RS:2;d7d59561a64e:36243 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36243,1732134919381/d7d59561a64e%2C36243%2C1732134919381.1732134922362, exclude list is [], retry=0 2024-11-20T20:35:22,420 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T20:35:22,421 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T20:35:22,428 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T20:35:22,429 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:22,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:22,431 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T20:35:22,431 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T20:35:22,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:22,434 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740 2024-11-20T20:35:22,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:22,435 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740 2024-11-20T20:35:22,435 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:22,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:22,439 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T20:35:22,440 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T20:35:22,441 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T20:35:22,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:22,447 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T20:35:22,471 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T20:35:22,473 INFO [RS:1;d7d59561a64e:33553 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,33553,1732134919262/d7d59561a64e%2C33553%2C1732134919262.1732134922358 2024-11-20T20:35:22,477 DEBUG [RS:1;d7d59561a64e:33553 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38233:38233),(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:39759:39759)] 2024-11-20T20:35:22,478 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66294709, jitterRate=-0.012131854891777039}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T20:35:22,480 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732134922336Initializing all the Stores at 1732134922349 (+13 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732134922350 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732134922353 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134922353Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732134922353Cleaning up temporary data from old regions at 1732134922440 (+87 ms)Region opened successfully at 1732134922480 (+40 ms) 2024-11-20T20:35:22,480 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T20:35:22,480 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T20:35:22,480 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T20:35:22,480 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T20:35:22,481 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T20:35:22,501 INFO [RS:2;d7d59561a64e:36243 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36243,1732134919381/d7d59561a64e%2C36243%2C1732134919381.1732134922362 2024-11-20T20:35:22,503 INFO [RS:0;d7d59561a64e:36639 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021/d7d59561a64e%2C36639%2C1732134919021.1732134922368 2024-11-20T20:35:22,504 DEBUG [RS:2;d7d59561a64e:36243 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:39759:39759),(127.0.0.1/127.0.0.1:38233:38233)] 2024-11-20T20:35:22,507 DEBUG [RS:0;d7d59561a64e:36639 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38233:38233),(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:39759:39759)] 2024-11-20T20:35:22,508 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T20:35:22,508 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732134922480Disabling compacts and flushes for region at 1732134922480Disabling writes for close at 1732134922481 (+1 ms)Writing region close event to WAL at 1732134922507 (+26 ms)Closed at 1732134922508 (+1 ms) 2024-11-20T20:35:22,517 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T20:35:22,517 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T20:35:22,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T20:35:22,536 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T20:35:22,540 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T20:35:22,693 DEBUG [d7d59561a64e:46641 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-20T20:35:22,700 DEBUG [d7d59561a64e:46641 {}] balancer.BalancerClusterState(204): Hosts are {d7d59561a64e=0} racks are {/default-rack=0} 2024-11-20T20:35:22,707 DEBUG [d7d59561a64e:46641 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T20:35:22,707 DEBUG [d7d59561a64e:46641 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T20:35:22,707 DEBUG [d7d59561a64e:46641 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-20T20:35:22,707 DEBUG [d7d59561a64e:46641 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T20:35:22,707 DEBUG [d7d59561a64e:46641 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T20:35:22,707 DEBUG [d7d59561a64e:46641 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-20T20:35:22,707 INFO [d7d59561a64e:46641 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T20:35:22,707 INFO [d7d59561a64e:46641 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T20:35:22,708 INFO [d7d59561a64e:46641 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-20T20:35:22,708 DEBUG [d7d59561a64e:46641 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T20:35:22,716 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d7d59561a64e,36639,1732134919021 2024-11-20T20:35:22,721 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d7d59561a64e,36639,1732134919021, state=OPENING 2024-11-20T20:35:22,788 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T20:35:22,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:35:22,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33553-0x10159f54fc60002, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:35:22,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36639-0x10159f54fc60001, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:35:22,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36243-0x10159f54fc60003, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:35:22,800 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T20:35:22,800 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T20:35:22,800 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T20:35:22,800 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T20:35:22,802 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T20:35:22,805 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d7d59561a64e,36639,1732134919021}] 2024-11-20T20:35:22,983 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T20:35:22,985 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45431, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T20:35:22,998 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T20:35:22,999 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:22,999 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T20:35:23,003 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d7d59561a64e%2C36639%2C1732134919021.meta, suffix=.meta, logDir=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021, archiveDir=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/oldWALs, maxLogs=32 2024-11-20T20:35:23,020 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021/d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta, exclude list is [], retry=0 2024-11-20T20:35:23,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:23,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:23,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:23,027 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021/d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta 2024-11-20T20:35:23,027 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38233:38233),(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:39759:39759)] 2024-11-20T20:35:23,028 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:35:23,030 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T20:35:23,032 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T20:35:23,038 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T20:35:23,043 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T20:35:23,044 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:23,044 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T20:35:23,044 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T20:35:23,047 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T20:35:23,049 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T20:35:23,049 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:23,050 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T20:35:23,050 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T20:35:23,051 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T20:35:23,052 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:23,052 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T20:35:23,052 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T20:35:23,054 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T20:35:23,054 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:23,055 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T20:35:23,055 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T20:35:23,056 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T20:35:23,056 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:23,057 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T20:35:23,057 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T20:35:23,059 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740 2024-11-20T20:35:23,062 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740 2024-11-20T20:35:23,064 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T20:35:23,064 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T20:35:23,065 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T20:35:23,068 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T20:35:23,069 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63150752, jitterRate=-0.058980464935302734}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T20:35:23,070 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T20:35:23,071 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732134923045Writing region info on filesystem at 1732134923045Initializing all the Stores at 1732134923047 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732134923047Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732134923047Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134923047Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732134923047Cleaning up temporary data from old regions at 1732134923064 (+17 ms)Running coprocessor post-open hooks at 1732134923070 (+6 ms)Region opened successfully at 1732134923071 (+1 ms) 2024-11-20T20:35:23,079 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732134922973 2024-11-20T20:35:23,089 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T20:35:23,090 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T20:35:23,092 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d7d59561a64e,36639,1732134919021 2024-11-20T20:35:23,094 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d7d59561a64e,36639,1732134919021, state=OPEN 2024-11-20T20:35:23,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T20:35:23,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36639-0x10159f54fc60001, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T20:35:23,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36243-0x10159f54fc60003, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T20:35:23,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33553-0x10159f54fc60002, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T20:35:23,195 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T20:35:23,195 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T20:35:23,195 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T20:35:23,195 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T20:35:23,196 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d7d59561a64e,36639,1732134919021 2024-11-20T20:35:23,201 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T20:35:23,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d7d59561a64e,36639,1732134919021 in 392 msec 2024-11-20T20:35:23,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T20:35:23,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 678 msec 2024-11-20T20:35:23,211 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T20:35:23,211 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T20:35:23,232 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T20:35:23,233 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,36639,1732134919021, seqNum=-1] 2024-11-20T20:35:23,256 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T20:35:23,258 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56427, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T20:35:23,279 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6880 sec 2024-11-20T20:35:23,279 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732134923279, completionTime=-1 2024-11-20T20:35:23,283 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-20T20:35:23,283 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T20:35:23,310 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-20T20:35:23,311 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732134983311 2024-11-20T20:35:23,311 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732135043311 2024-11-20T20:35:23,311 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 27 msec 2024-11-20T20:35:23,313 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-20T20:35:23,345 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d7d59561a64e,46641,1732134918030-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:23,345 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d7d59561a64e,46641,1732134918030-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:23,345 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d7d59561a64e,46641,1732134918030-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:23,347 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d7d59561a64e:46641, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:23,348 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:23,348 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:23,356 DEBUG [master/d7d59561a64e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T20:35:23,391 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.756sec 2024-11-20T20:35:23,393 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T20:35:23,394 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T20:35:23,396 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T20:35:23,396 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T20:35:23,396 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T20:35:23,397 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d7d59561a64e,46641,1732134918030-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T20:35:23,398 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d7d59561a64e,46641,1732134918030-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T20:35:23,404 DEBUG [master/d7d59561a64e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T20:35:23,405 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T20:35:23,406 INFO [master/d7d59561a64e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d7d59561a64e,46641,1732134918030-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:23,468 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4136470, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T20:35:23,470 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d7d59561a64e,46641,-1 for getting cluster id 2024-11-20T20:35:23,474 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T20:35:23,486 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '92b6d327-3593-434a-bf0b-fee82840fcda' 2024-11-20T20:35:23,491 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T20:35:23,491 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "92b6d327-3593-434a-bf0b-fee82840fcda" 2024-11-20T20:35:23,492 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33307f93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T20:35:23,492 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d7d59561a64e,46641,-1] 2024-11-20T20:35:23,499 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T20:35:23,501 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T20:35:23,504 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42076, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T20:35:23,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34f3e6a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T20:35:23,510 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T20:35:23,525 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,36639,1732134919021, seqNum=-1] 2024-11-20T20:35:23,526 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T20:35:23,533 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T20:35:23,565 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d7d59561a64e,46641,1732134918030 2024-11-20T20:35:23,565 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:33605/hbase 2024-11-20T20:35:23,585 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=362, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=945, ProcessCount=12, AvailableMemoryMB=2219 2024-11-20T20:35:23,606 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:23,610 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:23,612 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T20:35:23,617 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-08598763, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/hregion-08598763, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:23,636 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-08598763/hregion-08598763.1732134923618, exclude list is [], retry=0 2024-11-20T20:35:23,645 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:23,661 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:23,664 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:23,673 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-08598763/hregion-08598763.1732134923618 2024-11-20T20:35:23,677 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39759:39759),(127.0.0.1/127.0.0.1:38233:38233),(127.0.0.1/127.0.0.1:46511:46511)] 2024-11-20T20:35:23,677 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4cec03eb4538b2cc241fdb2befe55f8b, NAME => 'testReplayEditsWrittenIntoWAL,,1732134923607.4cec03eb4538b2cc241fdb2befe55f8b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33605/hbase 2024-11-20T20:35:23,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741838_1014 (size=64) 2024-11-20T20:35:23,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741838_1014 (size=64) 2024-11-20T20:35:23,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741838_1014 (size=64) 2024-11-20T20:35:23,742 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732134923607.4cec03eb4538b2cc241fdb2befe55f8b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:23,764 INFO [StoreOpener-4cec03eb4538b2cc241fdb2befe55f8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4cec03eb4538b2cc241fdb2befe55f8b 2024-11-20T20:35:23,768 INFO [StoreOpener-4cec03eb4538b2cc241fdb2befe55f8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4cec03eb4538b2cc241fdb2befe55f8b columnFamilyName a 2024-11-20T20:35:23,769 DEBUG [StoreOpener-4cec03eb4538b2cc241fdb2befe55f8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:23,770 INFO [StoreOpener-4cec03eb4538b2cc241fdb2befe55f8b-1 {}] regionserver.HStore(327): Store=4cec03eb4538b2cc241fdb2befe55f8b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:23,770 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4cec03eb4538b2cc241fdb2befe55f8b 2024-11-20T20:35:23,773 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b 2024-11-20T20:35:23,774 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b 2024-11-20T20:35:23,775 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4cec03eb4538b2cc241fdb2befe55f8b 2024-11-20T20:35:23,775 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4cec03eb4538b2cc241fdb2befe55f8b 2024-11-20T20:35:23,780 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4cec03eb4538b2cc241fdb2befe55f8b 2024-11-20T20:35:23,791 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T20:35:23,796 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4cec03eb4538b2cc241fdb2befe55f8b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62389467, jitterRate=-0.07032449543476105}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T20:35:23,797 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4cec03eb4538b2cc241fdb2befe55f8b: Writing region info on filesystem at 1732134923742Initializing all the Stores at 1732134923751 (+9 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134923751Cleaning up temporary data from old regions at 1732134923775 (+24 ms)Region opened successfully at 1732134923797 (+22 ms) 2024-11-20T20:35:23,798 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4cec03eb4538b2cc241fdb2befe55f8b, disabling compactions & flushes 2024-11-20T20:35:23,798 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732134923607.4cec03eb4538b2cc241fdb2befe55f8b. 2024-11-20T20:35:23,798 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732134923607.4cec03eb4538b2cc241fdb2befe55f8b. 2024-11-20T20:35:23,798 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732134923607.4cec03eb4538b2cc241fdb2befe55f8b. after waiting 0 ms 2024-11-20T20:35:23,798 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732134923607.4cec03eb4538b2cc241fdb2befe55f8b. 2024-11-20T20:35:23,802 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732134923607.4cec03eb4538b2cc241fdb2befe55f8b. 2024-11-20T20:35:23,802 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4cec03eb4538b2cc241fdb2befe55f8b: Waiting for close lock at 1732134923798Disabling compacts and flushes for region at 1732134923798Disabling writes for close at 1732134923798Writing region close event to WAL at 1732134923802 (+4 ms)Closed at 1732134923802 2024-11-20T20:35:23,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741837_1013 (size=93) 2024-11-20T20:35:23,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741837_1013 (size=93) 2024-11-20T20:35:23,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741837_1013 (size=93) 2024-11-20T20:35:23,830 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T20:35:23,831 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-08598763:(num 1732134923618) 2024-11-20T20:35:23,833 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-20T20:35:23,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741839_1015 (size=276) 2024-11-20T20:35:23,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741839_1015 (size=276) 2024-11-20T20:35:23,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741839_1015 (size=276) 2024-11-20T20:35:23,851 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-20T20:35:23,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741840_1016 (size=230) 2024-11-20T20:35:23,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741840_1016 (size=230) 2024-11-20T20:35:23,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741840_1016 (size=230) 2024-11-20T20:35:23,885 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal-1, size=276 (276bytes) 2024-11-20T20:35:23,885 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-20T20:35:23,885 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-20T20:35:23,885 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal-1 2024-11-20T20:35:23,891 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal-1 after 3ms 2024-11-20T20:35:23,896 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:23,897 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal-1 took 14ms 2024-11-20T20:35:23,902 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal-1 so closing down 2024-11-20T20:35:23,905 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-20T20:35:23,907 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T20:35:23,907 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b/recovered.edits/0000000000000000001-wal-1.temp 2024-11-20T20:35:23,909 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T20:35:23,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741841_1017 (size=276) 2024-11-20T20:35:23,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741841_1017 (size=276) 2024-11-20T20:35:23,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741841_1017 (size=276) 2024-11-20T20:35:23,920 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-20T20:35:23,924 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b/recovered.edits/0000000000000000002 2024-11-20T20:35:23,929 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-11-20T20:35:23,929 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal-1, journal: Splitting hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal-1, size=276 (276bytes) at 1732134923885Finishing writing output for hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal-1 so closing down at 1732134923902 (+17 ms)Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b/recovered.edits/0000000000000000001-wal-1.temp at 1732134923908 (+6 ms)3 split writer threads finished at 1732134923909 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732134923920 (+11 ms)Rename recovered edits hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b/recovered.edits/0000000000000000002 at 1732134923924 (+4 ms)Processed 2 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1732134923929 (+5 ms) 2024-11-20T20:35:23,946 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal-2, size=230 (230bytes) 2024-11-20T20:35:23,946 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal-2 2024-11-20T20:35:23,947 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal-2 after 1ms 2024-11-20T20:35:23,951 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:23,951 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal-2 took 5ms 2024-11-20T20:35:23,953 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal-2 so closing down 2024-11-20T20:35:23,954 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T20:35:23,957 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-20T20:35:23,958 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b/recovered.edits/0000000000000000002-wal-2.temp 2024-11-20T20:35:23,958 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T20:35:23,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741842_1018 (size=230) 2024-11-20T20:35:23,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741842_1018 (size=230) 2024-11-20T20:35:23,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741842_1018 (size=230) 2024-11-20T20:35:23,968 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-20T20:35:23,973 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:23,976 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b/recovered.edits/0000000000000000002-wal-2.temp, length=230 2024-11-20T20:35:23,977 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-11-20T20:35:23,978 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal-2, journal: Splitting hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal-2, size=230 (230bytes) at 1732134923946Finishing writing output for hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal-2 so closing down at 1732134923954 (+8 ms)Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b/recovered.edits/0000000000000000002-wal-2.temp at 1732134923958 (+4 ms)3 split writer threads finished at 1732134923958Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732134923968 (+10 ms)Processed 1 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1732134923977 (+9 ms) 2024-11-20T20:35:23,978 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T20:35:23,981 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:23,995 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal.1732134923982, exclude list is [], retry=0 2024-11-20T20:35:24,000 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:24,001 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:24,001 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:24,003 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal.1732134923982 2024-11-20T20:35:24,004 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39759:39759),(127.0.0.1/127.0.0.1:38233:38233),(127.0.0.1/127.0.0.1:46511:46511)] 2024-11-20T20:35:24,004 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4cec03eb4538b2cc241fdb2befe55f8b, NAME => 'testReplayEditsWrittenIntoWAL,,1732134923607.4cec03eb4538b2cc241fdb2befe55f8b.', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:35:24,004 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732134923607.4cec03eb4538b2cc241fdb2befe55f8b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:24,004 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4cec03eb4538b2cc241fdb2befe55f8b 2024-11-20T20:35:24,004 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4cec03eb4538b2cc241fdb2befe55f8b 2024-11-20T20:35:24,006 INFO [StoreOpener-4cec03eb4538b2cc241fdb2befe55f8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4cec03eb4538b2cc241fdb2befe55f8b 2024-11-20T20:35:24,008 INFO [StoreOpener-4cec03eb4538b2cc241fdb2befe55f8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4cec03eb4538b2cc241fdb2befe55f8b columnFamilyName a 2024-11-20T20:35:24,008 DEBUG [StoreOpener-4cec03eb4538b2cc241fdb2befe55f8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:24,009 INFO [StoreOpener-4cec03eb4538b2cc241fdb2befe55f8b-1 {}] regionserver.HStore(327): Store=4cec03eb4538b2cc241fdb2befe55f8b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:24,009 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4cec03eb4538b2cc241fdb2befe55f8b 2024-11-20T20:35:24,010 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b 2024-11-20T20:35:24,013 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b 2024-11-20T20:35:24,015 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b/recovered.edits/0000000000000000002 2024-11-20T20:35:24,018 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:24,025 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b/recovered.edits/0000000000000000002 2024-11-20T20:35:24,029 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4cec03eb4538b2cc241fdb2befe55f8b 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-20T20:35:24,094 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b/.tmp/a/d823a386411a446bb39cb448fd324ada is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732134923831/Put/seqid=0 2024-11-20T20:35:24,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741844_1020 (size=5170) 2024-11-20T20:35:24,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741844_1020 (size=5170) 2024-11-20T20:35:24,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741844_1020 (size=5170) 2024-11-20T20:35:24,108 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b/.tmp/a/d823a386411a446bb39cb448fd324ada 2024-11-20T20:35:24,150 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b/.tmp/a/d823a386411a446bb39cb448fd324ada as hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b/a/d823a386411a446bb39cb448fd324ada 2024-11-20T20:35:24,160 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b/a/d823a386411a446bb39cb448fd324ada, entries=2, sequenceid=2, filesize=5.0 K 2024-11-20T20:35:24,165 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 4cec03eb4538b2cc241fdb2befe55f8b in 135ms, sequenceid=2, compaction requested=false; wal=null 2024-11-20T20:35:24,167 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b/recovered.edits/0000000000000000002 2024-11-20T20:35:24,168 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4cec03eb4538b2cc241fdb2befe55f8b 2024-11-20T20:35:24,168 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4cec03eb4538b2cc241fdb2befe55f8b 2024-11-20T20:35:24,171 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4cec03eb4538b2cc241fdb2befe55f8b 2024-11-20T20:35:24,175 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/4cec03eb4538b2cc241fdb2befe55f8b/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-20T20:35:24,177 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4cec03eb4538b2cc241fdb2befe55f8b; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70248485, jitterRate=0.04678399860858917}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T20:35:24,178 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4cec03eb4538b2cc241fdb2befe55f8b: Writing region info on filesystem at 1732134924004Initializing all the Stores at 1732134924006 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134924006Obtaining lock to block concurrent updates at 1732134924029 (+23 ms)Preparing flush snapshotting stores in 4cec03eb4538b2cc241fdb2befe55f8b at 1732134924029Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732134923607.4cec03eb4538b2cc241fdb2befe55f8b., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732134924034 (+5 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732134923607.4cec03eb4538b2cc241fdb2befe55f8b. at 1732134924034Flushing 4cec03eb4538b2cc241fdb2befe55f8b/a: creating writer at 1732134924036 (+2 ms)Flushing 4cec03eb4538b2cc241fdb2befe55f8b/a: appending metadata at 1732134924082 (+46 ms)Flushing 4cec03eb4538b2cc241fdb2befe55f8b/a: closing flushed file at 1732134924084 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@733d7d22: reopening flushed file at 1732134924148 (+64 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 4cec03eb4538b2cc241fdb2befe55f8b in 135ms, sequenceid=2, compaction requested=false; wal=null at 1732134924165 (+17 ms)Cleaning up temporary data from old regions at 1732134924168 (+3 ms)Region opened successfully at 1732134924178 (+10 ms) 2024-11-20T20:35:24,207 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=374 (was 362) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:55992 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:51050 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:51024 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1180650432-172.17.0.2-1732134912528:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1180650432-172.17.0.2-1732134912528:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:41254 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:56050 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:33605/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1180650432-172.17.0.2-1732134912528:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:41280 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=945 (was 945), ProcessCount=11 (was 12), AvailableMemoryMB=2242 (was 2219) - AvailableMemoryMB LEAK? - 2024-11-20T20:35:24,220 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=374, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=945, ProcessCount=11, AvailableMemoryMB=2240 2024-11-20T20:35:24,234 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:24,237 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:24,238 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T20:35:24,244 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-62807721, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/hregion-62807721, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:24,260 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-62807721/hregion-62807721.1732134924245, exclude list is [], retry=0 2024-11-20T20:35:24,264 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:24,265 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:24,267 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:24,273 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-62807721/hregion-62807721.1732134924245 2024-11-20T20:35:24,273 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38233:38233),(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:39759:39759)] 2024-11-20T20:35:24,274 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 27d31ad369db1a3e57e8a58aaf35bb36, NAME => 'testReplayEditsWrittenIntoWAL,,1732134924235.27d31ad369db1a3e57e8a58aaf35bb36.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33605/hbase 2024-11-20T20:35:24,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741846_1022 (size=64) 2024-11-20T20:35:24,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741846_1022 (size=64) 2024-11-20T20:35:24,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741846_1022 (size=64) 2024-11-20T20:35:24,294 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732134924235.27d31ad369db1a3e57e8a58aaf35bb36.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:24,297 INFO [StoreOpener-27d31ad369db1a3e57e8a58aaf35bb36-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 27d31ad369db1a3e57e8a58aaf35bb36 2024-11-20T20:35:24,300 INFO [StoreOpener-27d31ad369db1a3e57e8a58aaf35bb36-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27d31ad369db1a3e57e8a58aaf35bb36 columnFamilyName a 2024-11-20T20:35:24,301 DEBUG [StoreOpener-27d31ad369db1a3e57e8a58aaf35bb36-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:24,302 INFO [StoreOpener-27d31ad369db1a3e57e8a58aaf35bb36-1 {}] regionserver.HStore(327): Store=27d31ad369db1a3e57e8a58aaf35bb36/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:24,302 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 27d31ad369db1a3e57e8a58aaf35bb36 2024-11-20T20:35:24,305 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36 2024-11-20T20:35:24,305 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36 2024-11-20T20:35:24,306 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 27d31ad369db1a3e57e8a58aaf35bb36 2024-11-20T20:35:24,306 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 27d31ad369db1a3e57e8a58aaf35bb36 2024-11-20T20:35:24,311 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 27d31ad369db1a3e57e8a58aaf35bb36 2024-11-20T20:35:24,316 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T20:35:24,317 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 27d31ad369db1a3e57e8a58aaf35bb36; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70975809, jitterRate=0.057621970772743225}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T20:35:24,317 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 27d31ad369db1a3e57e8a58aaf35bb36: Writing region info on filesystem at 1732134924294Initializing all the Stores at 1732134924296 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134924296Cleaning up temporary data from old regions at 1732134924306 (+10 ms)Region opened successfully at 1732134924317 (+11 ms) 2024-11-20T20:35:24,318 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 27d31ad369db1a3e57e8a58aaf35bb36, disabling compactions & flushes 2024-11-20T20:35:24,318 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732134924235.27d31ad369db1a3e57e8a58aaf35bb36. 2024-11-20T20:35:24,318 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732134924235.27d31ad369db1a3e57e8a58aaf35bb36. 2024-11-20T20:35:24,318 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732134924235.27d31ad369db1a3e57e8a58aaf35bb36. after waiting 0 ms 2024-11-20T20:35:24,318 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732134924235.27d31ad369db1a3e57e8a58aaf35bb36. 2024-11-20T20:35:24,320 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732134924235.27d31ad369db1a3e57e8a58aaf35bb36. 2024-11-20T20:35:24,320 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 27d31ad369db1a3e57e8a58aaf35bb36: Waiting for close lock at 1732134924318Disabling compacts and flushes for region at 1732134924318Disabling writes for close at 1732134924318Writing region close event to WAL at 1732134924320 (+2 ms)Closed at 1732134924320 2024-11-20T20:35:24,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741845_1021 (size=93) 2024-11-20T20:35:24,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741845_1021 (size=93) 2024-11-20T20:35:24,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741845_1021 (size=93) 2024-11-20T20:35:24,337 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T20:35:24,337 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-62807721:(num 1732134924245) 2024-11-20T20:35:24,338 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-20T20:35:24,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741847_1023 (size=276) 2024-11-20T20:35:24,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741847_1023 (size=276) 2024-11-20T20:35:24,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741847_1023 (size=276) 2024-11-20T20:35:24,366 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-20T20:35:24,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741848_1024 (size=230) 2024-11-20T20:35:24,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741848_1024 (size=230) 2024-11-20T20:35:24,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741848_1024 (size=230) 2024-11-20T20:35:24,406 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal-2, size=230 (230bytes) 2024-11-20T20:35:24,406 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal-2 2024-11-20T20:35:24,407 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal-2 after 1ms 2024-11-20T20:35:24,412 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:24,412 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal-2 took 7ms 2024-11-20T20:35:24,425 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal-2 so closing down 2024-11-20T20:35:24,425 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T20:35:24,437 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-20T20:35:24,439 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/recovered.edits/0000000000000000002-wal-2.temp 2024-11-20T20:35:24,440 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T20:35:24,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741849_1025 (size=230) 2024-11-20T20:35:24,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741849_1025 (size=230) 2024-11-20T20:35:24,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741849_1025 (size=230) 2024-11-20T20:35:24,456 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-20T20:35:24,459 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/recovered.edits/0000000000000000002 2024-11-20T20:35:24,459 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 37 ms; skipped=0; WAL=hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-11-20T20:35:24,459 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal-2, journal: Splitting hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal-2, size=230 (230bytes) at 1732134924406Finishing writing output for hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal-2 so closing down at 1732134924425 (+19 ms)Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/recovered.edits/0000000000000000002-wal-2.temp at 1732134924439 (+14 ms)3 split writer threads finished at 1732134924440 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732134924456 (+16 ms)Rename recovered edits hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/recovered.edits/0000000000000000002 at 1732134924459 (+3 ms)Processed 1 edits across 1 Regions in 37 ms; skipped=0; WAL=hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1732134924459 2024-11-20T20:35:24,480 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal-1, size=276 (276bytes) 2024-11-20T20:35:24,481 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal-1 2024-11-20T20:35:24,482 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal-1 after 1ms 2024-11-20T20:35:24,486 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:24,486 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal-1 took 6ms 2024-11-20T20:35:24,492 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal-1 so closing down 2024-11-20T20:35:24,492 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T20:35:24,496 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-20T20:35:24,500 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/recovered.edits/0000000000000000001-wal-1.temp 2024-11-20T20:35:24,501 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T20:35:24,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741850_1026 (size=276) 2024-11-20T20:35:24,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741850_1026 (size=276) 2024-11-20T20:35:24,521 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-20T20:35:24,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741850_1026 (size=276) 2024-11-20T20:35:24,530 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:24,533 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/recovered.edits/0000000000000000002, length=230 2024-11-20T20:35:24,567 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/recovered.edits/0000000000000000002 2024-11-20T20:35:24,567 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 79 ms; skipped=0; WAL=hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-11-20T20:35:24,567 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal-1, journal: Splitting hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal-1, size=276 (276bytes) at 1732134924481Finishing writing output for hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal-1 so closing down at 1732134924492 (+11 ms)Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/recovered.edits/0000000000000000001-wal-1.temp at 1732134924500 (+8 ms)3 split writer threads finished at 1732134924501 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732134924521 (+20 ms)Rename recovered edits hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/recovered.edits/0000000000000000002 at 1732134924567 (+46 ms)Processed 2 edits across 1 Regions in 79 ms; skipped=0; WAL=hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1732134924567 2024-11-20T20:35:24,567 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T20:35:24,570 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:24,584 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal.1732134924570, exclude list is [], retry=0 2024-11-20T20:35:24,587 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:24,588 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:24,588 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:24,591 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal.1732134924570 2024-11-20T20:35:24,591 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39759:39759),(127.0.0.1/127.0.0.1:38233:38233),(127.0.0.1/127.0.0.1:46511:46511)] 2024-11-20T20:35:24,591 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 27d31ad369db1a3e57e8a58aaf35bb36, NAME => 'testReplayEditsWrittenIntoWAL,,1732134924235.27d31ad369db1a3e57e8a58aaf35bb36.', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:35:24,592 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732134924235.27d31ad369db1a3e57e8a58aaf35bb36.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:24,592 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 27d31ad369db1a3e57e8a58aaf35bb36 2024-11-20T20:35:24,592 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 27d31ad369db1a3e57e8a58aaf35bb36 2024-11-20T20:35:24,596 INFO [StoreOpener-27d31ad369db1a3e57e8a58aaf35bb36-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 27d31ad369db1a3e57e8a58aaf35bb36 2024-11-20T20:35:24,598 INFO [StoreOpener-27d31ad369db1a3e57e8a58aaf35bb36-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27d31ad369db1a3e57e8a58aaf35bb36 columnFamilyName a 2024-11-20T20:35:24,599 DEBUG [StoreOpener-27d31ad369db1a3e57e8a58aaf35bb36-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:24,599 INFO [StoreOpener-27d31ad369db1a3e57e8a58aaf35bb36-1 {}] regionserver.HStore(327): Store=27d31ad369db1a3e57e8a58aaf35bb36/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:24,600 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 27d31ad369db1a3e57e8a58aaf35bb36 2024-11-20T20:35:24,601 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36 2024-11-20T20:35:24,603 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36 2024-11-20T20:35:24,604 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/recovered.edits/0000000000000000002 2024-11-20T20:35:24,608 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:24,610 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/recovered.edits/0000000000000000002 2024-11-20T20:35:24,610 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 27d31ad369db1a3e57e8a58aaf35bb36 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-20T20:35:24,629 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/.tmp/a/532a90d97a634140aaa8ec6b329edc14 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732134924337/Put/seqid=0 2024-11-20T20:35:24,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741852_1028 (size=5170) 2024-11-20T20:35:24,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741852_1028 (size=5170) 2024-11-20T20:35:24,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741852_1028 (size=5170) 2024-11-20T20:35:24,646 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/.tmp/a/532a90d97a634140aaa8ec6b329edc14 2024-11-20T20:35:24,657 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/.tmp/a/532a90d97a634140aaa8ec6b329edc14 as hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/a/532a90d97a634140aaa8ec6b329edc14 2024-11-20T20:35:24,667 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/a/532a90d97a634140aaa8ec6b329edc14, entries=2, sequenceid=2, filesize=5.0 K 2024-11-20T20:35:24,667 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 27d31ad369db1a3e57e8a58aaf35bb36 in 57ms, sequenceid=2, compaction requested=false; wal=null 2024-11-20T20:35:24,669 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/recovered.edits/0000000000000000002 2024-11-20T20:35:24,673 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 27d31ad369db1a3e57e8a58aaf35bb36 2024-11-20T20:35:24,673 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 27d31ad369db1a3e57e8a58aaf35bb36 2024-11-20T20:35:24,676 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 27d31ad369db1a3e57e8a58aaf35bb36 2024-11-20T20:35:24,680 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/27d31ad369db1a3e57e8a58aaf35bb36/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-20T20:35:24,681 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 27d31ad369db1a3e57e8a58aaf35bb36; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61790289, jitterRate=-0.0792529433965683}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T20:35:24,681 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 27d31ad369db1a3e57e8a58aaf35bb36: Writing region info on filesystem at 1732134924592Initializing all the Stores at 1732134924593 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134924593Obtaining lock to block concurrent updates at 1732134924610 (+17 ms)Preparing flush snapshotting stores in 27d31ad369db1a3e57e8a58aaf35bb36 at 1732134924610Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732134924235.27d31ad369db1a3e57e8a58aaf35bb36., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732134924611 (+1 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732134924235.27d31ad369db1a3e57e8a58aaf35bb36. at 1732134924611Flushing 27d31ad369db1a3e57e8a58aaf35bb36/a: creating writer at 1732134924611Flushing 27d31ad369db1a3e57e8a58aaf35bb36/a: appending metadata at 1732134924627 (+16 ms)Flushing 27d31ad369db1a3e57e8a58aaf35bb36/a: closing flushed file at 1732134924627Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17f9801: reopening flushed file at 1732134924655 (+28 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 27d31ad369db1a3e57e8a58aaf35bb36 in 57ms, sequenceid=2, compaction requested=false; wal=null at 1732134924667 (+12 ms)Cleaning up temporary data from old regions at 1732134924673 (+6 ms)Region opened successfully at 1732134924681 (+8 ms) 2024-11-20T20:35:24,700 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=384 (was 374) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:55992 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:56128 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:51114 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:51024 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1180650432-172.17.0.2-1732134912528:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1180650432-172.17.0.2-1732134912528:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:41348 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:41280 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1180650432-172.17.0.2-1732134912528:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=945 (was 945), ProcessCount=11 (was 11), AvailableMemoryMB=2188 (was 2240) 2024-11-20T20:35:24,711 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=384, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=945, ProcessCount=11, AvailableMemoryMB=2187 2024-11-20T20:35:24,727 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:24,730 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:24,731 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T20:35:24,735 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-21155472, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/hregion-21155472, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:24,751 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-21155472/hregion-21155472.1732134924736, exclude list is [], retry=0 2024-11-20T20:35:24,756 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:24,760 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:24,763 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:24,770 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-21155472/hregion-21155472.1732134924736 2024-11-20T20:35:24,771 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38233:38233),(127.0.0.1/127.0.0.1:39759:39759),(127.0.0.1/127.0.0.1:46511:46511)] 2024-11-20T20:35:24,772 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e4f4b0e1a8a5ad9a5488595ab8684b1e, NAME => 'testReplayEditsWrittenIntoWAL,,1732134924728.e4f4b0e1a8a5ad9a5488595ab8684b1e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33605/hbase 2024-11-20T20:35:24,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741854_1030 (size=64) 2024-11-20T20:35:24,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741854_1030 (size=64) 2024-11-20T20:35:24,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741854_1030 (size=64) 2024-11-20T20:35:24,804 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732134924728.e4f4b0e1a8a5ad9a5488595ab8684b1e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:24,812 INFO [StoreOpener-e4f4b0e1a8a5ad9a5488595ab8684b1e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e4f4b0e1a8a5ad9a5488595ab8684b1e 2024-11-20T20:35:24,815 INFO [StoreOpener-e4f4b0e1a8a5ad9a5488595ab8684b1e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4f4b0e1a8a5ad9a5488595ab8684b1e columnFamilyName a 2024-11-20T20:35:24,816 DEBUG [StoreOpener-e4f4b0e1a8a5ad9a5488595ab8684b1e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:24,817 INFO [StoreOpener-e4f4b0e1a8a5ad9a5488595ab8684b1e-1 {}] regionserver.HStore(327): Store=e4f4b0e1a8a5ad9a5488595ab8684b1e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:24,817 INFO [StoreOpener-e4f4b0e1a8a5ad9a5488595ab8684b1e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e4f4b0e1a8a5ad9a5488595ab8684b1e 2024-11-20T20:35:24,825 INFO [StoreOpener-e4f4b0e1a8a5ad9a5488595ab8684b1e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4f4b0e1a8a5ad9a5488595ab8684b1e columnFamilyName b 2024-11-20T20:35:24,825 DEBUG [StoreOpener-e4f4b0e1a8a5ad9a5488595ab8684b1e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:24,826 INFO [StoreOpener-e4f4b0e1a8a5ad9a5488595ab8684b1e-1 {}] regionserver.HStore(327): Store=e4f4b0e1a8a5ad9a5488595ab8684b1e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:24,826 INFO [StoreOpener-e4f4b0e1a8a5ad9a5488595ab8684b1e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e4f4b0e1a8a5ad9a5488595ab8684b1e 2024-11-20T20:35:24,829 INFO [StoreOpener-e4f4b0e1a8a5ad9a5488595ab8684b1e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4f4b0e1a8a5ad9a5488595ab8684b1e columnFamilyName c 2024-11-20T20:35:24,829 DEBUG [StoreOpener-e4f4b0e1a8a5ad9a5488595ab8684b1e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:24,830 INFO [StoreOpener-e4f4b0e1a8a5ad9a5488595ab8684b1e-1 {}] regionserver.HStore(327): Store=e4f4b0e1a8a5ad9a5488595ab8684b1e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:24,830 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e4f4b0e1a8a5ad9a5488595ab8684b1e 2024-11-20T20:35:24,831 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e 2024-11-20T20:35:24,832 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e 2024-11-20T20:35:24,834 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e4f4b0e1a8a5ad9a5488595ab8684b1e 2024-11-20T20:35:24,834 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e4f4b0e1a8a5ad9a5488595ab8684b1e 2024-11-20T20:35:24,835 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T20:35:24,838 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e4f4b0e1a8a5ad9a5488595ab8684b1e 2024-11-20T20:35:24,842 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T20:35:24,843 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e4f4b0e1a8a5ad9a5488595ab8684b1e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62794254, jitterRate=-0.06429269909858704}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T20:35:24,844 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e4f4b0e1a8a5ad9a5488595ab8684b1e: Writing region info on filesystem at 1732134924804Initializing all the Stores at 1732134924806 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134924806Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134924812 (+6 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134924812Cleaning up temporary data from old regions at 1732134924834 (+22 ms)Region opened successfully at 1732134924844 (+10 ms) 2024-11-20T20:35:24,844 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e4f4b0e1a8a5ad9a5488595ab8684b1e, disabling compactions & flushes 2024-11-20T20:35:24,844 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732134924728.e4f4b0e1a8a5ad9a5488595ab8684b1e. 2024-11-20T20:35:24,845 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732134924728.e4f4b0e1a8a5ad9a5488595ab8684b1e. 2024-11-20T20:35:24,845 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732134924728.e4f4b0e1a8a5ad9a5488595ab8684b1e. after waiting 0 ms 2024-11-20T20:35:24,845 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732134924728.e4f4b0e1a8a5ad9a5488595ab8684b1e. 2024-11-20T20:35:24,848 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732134924728.e4f4b0e1a8a5ad9a5488595ab8684b1e. 2024-11-20T20:35:24,849 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e4f4b0e1a8a5ad9a5488595ab8684b1e: Waiting for close lock at 1732134924844Disabling compacts and flushes for region at 1732134924844Disabling writes for close at 1732134924845 (+1 ms)Writing region close event to WAL at 1732134924848 (+3 ms)Closed at 1732134924848 2024-11-20T20:35:24,862 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-21155472/hregion-21155472.1732134924736 not finished, retry = 0 2024-11-20T20:35:24,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741853_1029 (size=93) 2024-11-20T20:35:24,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741853_1029 (size=93) 2024-11-20T20:35:24,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741853_1029 (size=93) 2024-11-20T20:35:24,967 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T20:35:24,967 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-21155472:(num 1732134924736) 2024-11-20T20:35:24,967 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T20:35:24,971 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732134924726, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:24,989 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732134924726/wal.1732134924971, exclude list is [], retry=0 2024-11-20T20:35:24,994 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:24,995 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:24,996 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:25,001 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732134924726/wal.1732134924971 2024-11-20T20:35:25,001 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38233:38233),(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:39759:39759)] 2024-11-20T20:35:25,449 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732134924726/wal.1732134924971, size=0 (0bytes) 2024-11-20T20:35:25,449 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732134924726/wal.1732134924971 might be still open, length is 0 2024-11-20T20:35:25,449 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732134924726/wal.1732134924971 2024-11-20T20:35:25,451 WARN [IPC Server handler 2 on default port 33605 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732134924726/wal.1732134924971 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-20T20:35:25,451 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732134924726/wal.1732134924971 after 2ms 2024-11-20T20:35:26,029 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:41388 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:35033:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41388 dst: /127.0.0.1:35033 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35033 remote=/127.0.0.1:41388]. Total timeout mills is 60000, 59323 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:35:26,030 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:56166 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:44519:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56166 dst: /127.0.0.1:44519 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:35:26,030 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:51154 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:40323:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51154 dst: /127.0.0.1:40323 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:35:26,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741855_1032 (size=200589) 2024-11-20T20:35:26,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741855_1032 (size=200589) 2024-11-20T20:35:26,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741855_1032 (size=200589) 2024-11-20T20:35:28,560 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T20:35:28,645 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T20:35:28,647 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-20T20:35:28,648 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-20T20:35:28,650 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T20:35:28,650 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-20T20:35:28,650 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-20T20:35:28,650 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-20T20:35:28,651 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-20T20:35:28,651 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-20T20:35:29,455 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732134924726/wal.1732134924971 after 4006ms 2024-11-20T20:35:29,489 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732134924726/wal.1732134924971: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:29,504 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732134924726/wal.1732134924971 took 4056ms 2024-11-20T20:35:29,519 INFO [Time-limited test-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732134924971.temp 2024-11-20T20:35:29,536 INFO [Time-limited test-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/recovered.edits/0000000000000000001-wal.1732134924971.temp 2024-11-20T20:35:29,652 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732134924726/wal.1732134924971; continuing. 2024-11-20T20:35:29,652 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732134924726/wal.1732134924971 so closing down 2024-11-20T20:35:29,652 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T20:35:29,657 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T20:35:29,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741856_1033 (size=200597) 2024-11-20T20:35:29,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741856_1033 (size=200597) 2024-11-20T20:35:29,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741856_1033 (size=200597) 2024-11-20T20:35:29,662 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/recovered.edits/0000000000000000001-wal.1732134924971.temp (wrote 3002 edits, skipped 0 edits in 45 ms) 2024-11-20T20:35:29,664 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/recovered.edits/0000000000000000001-wal.1732134924971.temp to hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/recovered.edits/0000000000000003002 2024-11-20T20:35:29,664 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 156 ms; skipped=0; WAL=hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732134924726/wal.1732134924971, size=0, length=0, corrupted=false, cancelled=false 2024-11-20T20:35:29,664 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732134924726/wal.1732134924971, journal: Splitting hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732134924726/wal.1732134924971, size=0 (0bytes) at 1732134925449Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/recovered.edits/0000000000000000001-wal.1732134924971.temp at 1732134929536 (+4087 ms)Split 1024 edits, skipped 0 edits. at 1732134929596 (+60 ms)Split 2048 edits, skipped 0 edits. at 1732134929626 (+30 ms)Finishing writing output for hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732134924726/wal.1732134924971 so closing down at 1732134929652 (+26 ms)3 split writer threads finished at 1732134929657 (+5 ms)Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/recovered.edits/0000000000000000001-wal.1732134924971.temp (wrote 3002 edits, skipped 0 edits in 45 ms) at 1732134929662 (+5 ms)Rename recovered edits hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/recovered.edits/0000000000000000001-wal.1732134924971.temp to hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/recovered.edits/0000000000000003002 at 1732134929664 (+2 ms)Processed 3002 edits across 1 Regions in 156 ms; skipped=0; WAL=hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732134924726/wal.1732134924971, size=0, length=0, corrupted=false, cancelled=false at 1732134929664 2024-11-20T20:35:29,682 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732134924726/wal.1732134924971 to hdfs://localhost:33605/hbase/oldWALs/wal.1732134924971 2024-11-20T20:35:29,689 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/recovered.edits/0000000000000003002 2024-11-20T20:35:29,689 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T20:35:29,704 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732134924726, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:29,728 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732134924726/wal.1732134929704, exclude list is [], retry=0 2024-11-20T20:35:29,735 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:29,736 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:29,736 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:29,751 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732134924726/wal.1732134929704 2024-11-20T20:35:29,752 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:38233:38233),(127.0.0.1/127.0.0.1:39759:39759)] 2024-11-20T20:35:29,752 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732134924728.e4f4b0e1a8a5ad9a5488595ab8684b1e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:29,760 INFO [StoreOpener-e4f4b0e1a8a5ad9a5488595ab8684b1e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e4f4b0e1a8a5ad9a5488595ab8684b1e 2024-11-20T20:35:29,762 INFO [StoreOpener-e4f4b0e1a8a5ad9a5488595ab8684b1e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4f4b0e1a8a5ad9a5488595ab8684b1e columnFamilyName a 2024-11-20T20:35:29,762 DEBUG [StoreOpener-e4f4b0e1a8a5ad9a5488595ab8684b1e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:29,769 INFO [StoreOpener-e4f4b0e1a8a5ad9a5488595ab8684b1e-1 {}] regionserver.HStore(327): Store=e4f4b0e1a8a5ad9a5488595ab8684b1e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:29,770 INFO [StoreOpener-e4f4b0e1a8a5ad9a5488595ab8684b1e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e4f4b0e1a8a5ad9a5488595ab8684b1e 2024-11-20T20:35:29,771 INFO [StoreOpener-e4f4b0e1a8a5ad9a5488595ab8684b1e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4f4b0e1a8a5ad9a5488595ab8684b1e columnFamilyName b 2024-11-20T20:35:29,771 DEBUG [StoreOpener-e4f4b0e1a8a5ad9a5488595ab8684b1e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:29,772 INFO [StoreOpener-e4f4b0e1a8a5ad9a5488595ab8684b1e-1 {}] regionserver.HStore(327): Store=e4f4b0e1a8a5ad9a5488595ab8684b1e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:29,772 INFO [StoreOpener-e4f4b0e1a8a5ad9a5488595ab8684b1e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e4f4b0e1a8a5ad9a5488595ab8684b1e 2024-11-20T20:35:29,776 INFO [StoreOpener-e4f4b0e1a8a5ad9a5488595ab8684b1e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4f4b0e1a8a5ad9a5488595ab8684b1e columnFamilyName c 2024-11-20T20:35:29,776 DEBUG [StoreOpener-e4f4b0e1a8a5ad9a5488595ab8684b1e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:29,777 INFO [StoreOpener-e4f4b0e1a8a5ad9a5488595ab8684b1e-1 {}] regionserver.HStore(327): Store=e4f4b0e1a8a5ad9a5488595ab8684b1e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:29,777 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e4f4b0e1a8a5ad9a5488595ab8684b1e 2024-11-20T20:35:29,779 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e 2024-11-20T20:35:29,782 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e 2024-11-20T20:35:29,784 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/recovered.edits/0000000000000003002 2024-11-20T20:35:29,788 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:29,868 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-20T20:35:30,523 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e4f4b0e1a8a5ad9a5488595ab8684b1e 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-20T20:35:30,612 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/.tmp/a/08a545e010eb4a919d448c3c8741922b is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732134925011/Put/seqid=0 2024-11-20T20:35:30,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741858_1035 (size=50463) 2024-11-20T20:35:30,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741858_1035 (size=50463) 2024-11-20T20:35:30,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741858_1035 (size=50463) 2024-11-20T20:35:30,657 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/.tmp/a/08a545e010eb4a919d448c3c8741922b 2024-11-20T20:35:30,690 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/.tmp/a/08a545e010eb4a919d448c3c8741922b as hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/a/08a545e010eb4a919d448c3c8741922b 2024-11-20T20:35:30,712 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/a/08a545e010eb4a919d448c3c8741922b, entries=754, sequenceid=754, filesize=49.3 K 2024-11-20T20:35:30,712 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for e4f4b0e1a8a5ad9a5488595ab8684b1e in 189ms, sequenceid=754, compaction requested=false; wal=null 2024-11-20T20:35:30,786 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-20T20:35:30,786 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e4f4b0e1a8a5ad9a5488595ab8684b1e 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-20T20:35:30,818 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/.tmp/a/5e25c337ad084dccb8f2f4607f41bc47 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732134925095/Put/seqid=0 2024-11-20T20:35:30,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741859_1036 (size=20072) 2024-11-20T20:35:30,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741859_1036 (size=20072) 2024-11-20T20:35:30,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741859_1036 (size=20072) 2024-11-20T20:35:30,876 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/.tmp/a/5e25c337ad084dccb8f2f4607f41bc47 2024-11-20T20:35:30,988 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/.tmp/b/59c121d5583442a3892353ab39adb4dd is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732134925135/Put/seqid=0 2024-11-20T20:35:31,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741860_1037 (size=35835) 2024-11-20T20:35:31,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741860_1037 (size=35835) 2024-11-20T20:35:31,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741860_1037 (size=35835) 2024-11-20T20:35:31,029 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/.tmp/b/59c121d5583442a3892353ab39adb4dd 2024-11-20T20:35:31,048 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/.tmp/a/5e25c337ad084dccb8f2f4607f41bc47 as hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/a/5e25c337ad084dccb8f2f4607f41bc47 2024-11-20T20:35:31,072 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/a/5e25c337ad084dccb8f2f4607f41bc47, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-20T20:35:31,074 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/.tmp/b/59c121d5583442a3892353ab39adb4dd as hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/b/59c121d5583442a3892353ab39adb4dd 2024-11-20T20:35:31,085 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/b/59c121d5583442a3892353ab39adb4dd, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-20T20:35:31,086 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for e4f4b0e1a8a5ad9a5488595ab8684b1e in 299ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-20T20:35:31,108 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-20T20:35:31,108 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e4f4b0e1a8a5ad9a5488595ab8684b1e 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-20T20:35:31,122 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/.tmp/b/39d762cb1df448969d29207d8c642242 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732134925165/Put/seqid=0 2024-11-20T20:35:31,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741861_1038 (size=35082) 2024-11-20T20:35:31,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741861_1038 (size=35082) 2024-11-20T20:35:31,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741861_1038 (size=35082) 2024-11-20T20:35:31,161 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/.tmp/b/39d762cb1df448969d29207d8c642242 2024-11-20T20:35:31,247 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/.tmp/c/3c7bc5323d704834b5daffe416c35791 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732134925237/Put/seqid=0 2024-11-20T20:35:31,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741862_1039 (size=20825) 2024-11-20T20:35:31,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741862_1039 (size=20825) 2024-11-20T20:35:31,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741862_1039 (size=20825) 2024-11-20T20:35:31,309 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/.tmp/c/3c7bc5323d704834b5daffe416c35791 2024-11-20T20:35:31,320 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/.tmp/b/39d762cb1df448969d29207d8c642242 as hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/b/39d762cb1df448969d29207d8c642242 2024-11-20T20:35:31,352 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/b/39d762cb1df448969d29207d8c642242, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-20T20:35:31,354 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/.tmp/c/3c7bc5323d704834b5daffe416c35791 as hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/c/3c7bc5323d704834b5daffe416c35791 2024-11-20T20:35:31,372 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/c/3c7bc5323d704834b5daffe416c35791, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-20T20:35:31,372 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for e4f4b0e1a8a5ad9a5488595ab8684b1e in 264ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-20T20:35:31,390 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732134925349/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732134924728.e4f4b0e1a8a5ad9a5488595ab8684b1e. 2024-11-20T20:35:31,397 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/recovered.edits/0000000000000003002 2024-11-20T20:35:31,397 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-20T20:35:31,397 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e4f4b0e1a8a5ad9a5488595ab8684b1e 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-20T20:35:31,427 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/.tmp/c/a9bf542743fd42958ba093cb73f78b69 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732134925255/Put/seqid=0 2024-11-20T20:35:31,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741863_1040 (size=50301) 2024-11-20T20:35:31,504 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/.tmp/c/a9bf542743fd42958ba093cb73f78b69 2024-11-20T20:35:31,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741863_1040 (size=50301) 2024-11-20T20:35:31,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741863_1040 (size=50301) 2024-11-20T20:35:31,514 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a9bf542743fd42958ba093cb73f78b69 2024-11-20T20:35:31,517 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/.tmp/c/a9bf542743fd42958ba093cb73f78b69 as hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/c/a9bf542743fd42958ba093cb73f78b69 2024-11-20T20:35:31,532 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a9bf542743fd42958ba093cb73f78b69 2024-11-20T20:35:31,533 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/c/a9bf542743fd42958ba093cb73f78b69, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-20T20:35:31,533 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for e4f4b0e1a8a5ad9a5488595ab8684b1e in 136ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-20T20:35:31,535 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/recovered.edits/0000000000000003002 2024-11-20T20:35:31,538 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e4f4b0e1a8a5ad9a5488595ab8684b1e 2024-11-20T20:35:31,538 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e4f4b0e1a8a5ad9a5488595ab8684b1e 2024-11-20T20:35:31,539 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T20:35:31,547 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e4f4b0e1a8a5ad9a5488595ab8684b1e 2024-11-20T20:35:31,573 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenIntoWAL/e4f4b0e1a8a5ad9a5488595ab8684b1e/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-20T20:35:31,585 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e4f4b0e1a8a5ad9a5488595ab8684b1e; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68806924, jitterRate=0.02530306577682495}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T20:35:31,585 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e4f4b0e1a8a5ad9a5488595ab8684b1e: Writing region info on filesystem at 1732134929752Initializing all the Stores at 1732134929757 (+5 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134929758 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134929760 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134929760Cleaning up temporary data from old regions at 1732134931538 (+1778 ms)Region opened successfully at 1732134931585 (+47 ms) 2024-11-20T20:35:31,695 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e4f4b0e1a8a5ad9a5488595ab8684b1e, disabling compactions & flushes 2024-11-20T20:35:31,695 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732134924728.e4f4b0e1a8a5ad9a5488595ab8684b1e. 2024-11-20T20:35:31,695 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732134924728.e4f4b0e1a8a5ad9a5488595ab8684b1e. 2024-11-20T20:35:31,695 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732134924728.e4f4b0e1a8a5ad9a5488595ab8684b1e. after waiting 0 ms 2024-11-20T20:35:31,695 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732134924728.e4f4b0e1a8a5ad9a5488595ab8684b1e. 2024-11-20T20:35:31,758 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732134924728.e4f4b0e1a8a5ad9a5488595ab8684b1e. 2024-11-20T20:35:31,758 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e4f4b0e1a8a5ad9a5488595ab8684b1e: Waiting for close lock at 1732134931695Disabling compacts and flushes for region at 1732134931695Disabling writes for close at 1732134931695Writing region close event to WAL at 1732134931758 (+63 ms)Closed at 1732134931758 2024-11-20T20:35:31,778 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732134924726/wal.1732134929704 not finished, retry = 0 2024-11-20T20:35:31,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741857_1034 (size=93) 2024-11-20T20:35:31,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741857_1034 (size=93) 2024-11-20T20:35:31,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741857_1034 (size=93) 2024-11-20T20:35:31,890 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T20:35:31,890 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732134929704) 2024-11-20T20:35:31,914 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=400 (was 384) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:46129 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/cluster_bbcbe4e8-087e-d341-dffc-b7e2617acb01/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/cluster_bbcbe4e8-087e-d341-dffc-b7e2617acb01/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46129 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/cluster_bbcbe4e8-087e-d341-dffc-b7e2617acb01/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46509 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:33605 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2138374406_22 at /127.0.0.1:56246 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2138374406_22 at /127.0.0.1:41430 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/cluster_bbcbe4e8-087e-d341-dffc-b7e2617acb01/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:46509 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2138374406_22 at /127.0.0.1:51182 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:33605 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2138374406_22 at /127.0.0.1:56220 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=933 (was 945), ProcessCount=11 (was 11), AvailableMemoryMB=2248 (was 2187) - AvailableMemoryMB LEAK? - 2024-11-20T20:35:31,934 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=400, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=933, ProcessCount=11, AvailableMemoryMB=2246 2024-11-20T20:35:31,958 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:31,963 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:31,964 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T20:35:31,972 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-31368440, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/hregion-31368440, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:31,996 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-31368440/hregion-31368440.1732134931972, exclude list is [], retry=0 2024-11-20T20:35:32,002 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:32,010 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:32,013 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:32,027 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-31368440/hregion-31368440.1732134931972 2024-11-20T20:35:32,028 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39759:39759),(127.0.0.1/127.0.0.1:38233:38233),(127.0.0.1/127.0.0.1:46511:46511)] 2024-11-20T20:35:32,028 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => bfa90791c9bfd148422bed44d5ff1248, NAME => 'test2727,,1732134931961.bfa90791c9bfd148422bed44d5ff1248.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33605/hbase 2024-11-20T20:35:32,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741865_1042 (size=43) 2024-11-20T20:35:32,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741865_1042 (size=43) 2024-11-20T20:35:32,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741865_1042 (size=43) 2024-11-20T20:35:32,057 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732134931961.bfa90791c9bfd148422bed44d5ff1248.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:32,061 INFO [StoreOpener-bfa90791c9bfd148422bed44d5ff1248-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bfa90791c9bfd148422bed44d5ff1248 2024-11-20T20:35:32,077 INFO [StoreOpener-bfa90791c9bfd148422bed44d5ff1248-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa90791c9bfd148422bed44d5ff1248 columnFamilyName a 2024-11-20T20:35:32,077 DEBUG [StoreOpener-bfa90791c9bfd148422bed44d5ff1248-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:32,079 INFO [StoreOpener-bfa90791c9bfd148422bed44d5ff1248-1 {}] regionserver.HStore(327): Store=bfa90791c9bfd148422bed44d5ff1248/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:32,079 INFO [StoreOpener-bfa90791c9bfd148422bed44d5ff1248-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bfa90791c9bfd148422bed44d5ff1248 2024-11-20T20:35:32,083 INFO [StoreOpener-bfa90791c9bfd148422bed44d5ff1248-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa90791c9bfd148422bed44d5ff1248 columnFamilyName b 2024-11-20T20:35:32,084 DEBUG [StoreOpener-bfa90791c9bfd148422bed44d5ff1248-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:32,092 INFO [StoreOpener-bfa90791c9bfd148422bed44d5ff1248-1 {}] regionserver.HStore(327): Store=bfa90791c9bfd148422bed44d5ff1248/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:32,093 INFO [StoreOpener-bfa90791c9bfd148422bed44d5ff1248-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bfa90791c9bfd148422bed44d5ff1248 2024-11-20T20:35:32,095 INFO [StoreOpener-bfa90791c9bfd148422bed44d5ff1248-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa90791c9bfd148422bed44d5ff1248 columnFamilyName c 2024-11-20T20:35:32,095 DEBUG [StoreOpener-bfa90791c9bfd148422bed44d5ff1248-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:32,096 INFO [StoreOpener-bfa90791c9bfd148422bed44d5ff1248-1 {}] regionserver.HStore(327): Store=bfa90791c9bfd148422bed44d5ff1248/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:32,097 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bfa90791c9bfd148422bed44d5ff1248 2024-11-20T20:35:32,104 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248 2024-11-20T20:35:32,106 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248 2024-11-20T20:35:32,109 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bfa90791c9bfd148422bed44d5ff1248 2024-11-20T20:35:32,109 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bfa90791c9bfd148422bed44d5ff1248 2024-11-20T20:35:32,110 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T20:35:32,111 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bfa90791c9bfd148422bed44d5ff1248 2024-11-20T20:35:32,121 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T20:35:32,124 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bfa90791c9bfd148422bed44d5ff1248; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61333649, jitterRate=-0.08605740964412689}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T20:35:32,126 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bfa90791c9bfd148422bed44d5ff1248: Writing region info on filesystem at 1732134932058Initializing all the Stores at 1732134932059 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134932059Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134932060 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134932060Cleaning up temporary data from old regions at 1732134932109 (+49 ms)Region opened successfully at 1732134932125 (+16 ms) 2024-11-20T20:35:32,126 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bfa90791c9bfd148422bed44d5ff1248, disabling compactions & flushes 2024-11-20T20:35:32,126 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732134931961.bfa90791c9bfd148422bed44d5ff1248. 2024-11-20T20:35:32,126 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732134931961.bfa90791c9bfd148422bed44d5ff1248. 2024-11-20T20:35:32,126 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732134931961.bfa90791c9bfd148422bed44d5ff1248. after waiting 0 ms 2024-11-20T20:35:32,126 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732134931961.bfa90791c9bfd148422bed44d5ff1248. 2024-11-20T20:35:32,129 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732134931961.bfa90791c9bfd148422bed44d5ff1248. 2024-11-20T20:35:32,130 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bfa90791c9bfd148422bed44d5ff1248: Waiting for close lock at 1732134932126Disabling compacts and flushes for region at 1732134932126Disabling writes for close at 1732134932126Writing region close event to WAL at 1732134932129 (+3 ms)Closed at 1732134932129 2024-11-20T20:35:32,140 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-31368440/hregion-31368440.1732134931972 not finished, retry = 0 2024-11-20T20:35:32,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741864_1041 (size=93) 2024-11-20T20:35:32,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741864_1041 (size=93) 2024-11-20T20:35:32,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741864_1041 (size=93) 2024-11-20T20:35:32,286 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T20:35:32,287 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-31368440:(num 1732134931972) 2024-11-20T20:35:32,287 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T20:35:32,302 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:32,333 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134932303, exclude list is [], retry=0 2024-11-20T20:35:32,345 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:32,345 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:32,346 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:32,373 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134932303 2024-11-20T20:35:32,376 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39759:39759),(127.0.0.1/127.0.0.1:38233:38233),(127.0.0.1/127.0.0.1:46511:46511)] 2024-11-20T20:35:32,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741866_1043 (size=200357) 2024-11-20T20:35:32,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741866_1043 (size=200357) 2024-11-20T20:35:32,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741866_1043 (size=200357) 2024-11-20T20:35:32,757 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134932303 not finished, retry = 0 2024-11-20T20:35:32,918 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134932303, size=195.7 K (200357bytes) 2024-11-20T20:35:32,918 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134932303 2024-11-20T20:35:32,919 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134932303 after 0ms 2024-11-20T20:35:32,928 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134932303: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:32,949 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134932303 took 32ms 2024-11-20T20:35:32,990 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732134932303.temp 2024-11-20T20:35:33,016 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000000001-wal.1732134932303.temp 2024-11-20T20:35:33,078 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134932303 so closing down 2024-11-20T20:35:33,079 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T20:35:33,084 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T20:35:33,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741867_1044 (size=200357) 2024-11-20T20:35:33,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741867_1044 (size=200357) 2024-11-20T20:35:33,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741867_1044 (size=200357) 2024-11-20T20:35:33,147 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000000001-wal.1732134932303.temp (wrote 3000 edits, skipped 0 edits in 22 ms) 2024-11-20T20:35:33,164 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000000001-wal.1732134932303.temp to hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000003000 2024-11-20T20:35:33,165 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 188 ms; skipped=0; WAL=hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134932303, size=195.7 K, length=200357, corrupted=false, cancelled=false 2024-11-20T20:35:33,165 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134932303, journal: Splitting hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134932303, size=195.7 K (200357bytes) at 1732134932918Split 1024 edits, skipped 0 edits. at 1732134933008 (+90 ms)Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000000001-wal.1732134932303.temp at 1732134933016 (+8 ms)Split 2048 edits, skipped 0 edits. at 1732134933049 (+33 ms)Finishing writing output for hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134932303 so closing down at 1732134933078 (+29 ms)3 split writer threads finished at 1732134933084 (+6 ms)Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000000001-wal.1732134932303.temp (wrote 3000 edits, skipped 0 edits in 22 ms) at 1732134933147 (+63 ms)Rename recovered edits hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000000001-wal.1732134932303.temp to hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000003000 at 1732134933164 (+17 ms)Processed 3000 edits across 1 Regions in 188 ms; skipped=0; WAL=hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134932303, size=195.7 K, length=200357, corrupted=false, cancelled=false at 1732134933165 (+1 ms) 2024-11-20T20:35:33,171 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134932303 to hdfs://localhost:33605/hbase/oldWALs/wal.1732134932303 2024-11-20T20:35:33,184 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000003000 2024-11-20T20:35:33,185 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T20:35:33,205 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:33,238 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134933206, exclude list is [], retry=0 2024-11-20T20:35:33,258 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:33,258 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:33,259 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:33,333 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134933206 2024-11-20T20:35:33,348 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:39759:39759),(127.0.0.1/127.0.0.1:38233:38233)] 2024-11-20T20:35:33,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741868_1045 (size=200484) 2024-11-20T20:35:33,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741868_1045 (size=200484) 2024-11-20T20:35:33,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741868_1045 (size=200484) 2024-11-20T20:35:33,715 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134933206, size=195.8 K (200484bytes) 2024-11-20T20:35:33,715 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134933206 2024-11-20T20:35:33,716 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134933206 after 1ms 2024-11-20T20:35:33,719 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134933206: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:33,727 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134933206 took 12ms 2024-11-20T20:35:33,745 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732134933206.temp 2024-11-20T20:35:33,755 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000003001-wal.1732134933206.temp 2024-11-20T20:35:33,807 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134933206 so closing down 2024-11-20T20:35:33,808 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T20:35:33,810 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T20:35:33,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741869_1046 (size=200484) 2024-11-20T20:35:33,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741869_1046 (size=200484) 2024-11-20T20:35:33,912 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000003001-wal.1732134933206.temp (wrote 3000 edits, skipped 0 edits in 20 ms) 2024-11-20T20:35:33,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741869_1046 (size=200484) 2024-11-20T20:35:33,915 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000003001-wal.1732134933206.temp to hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000006000 2024-11-20T20:35:33,916 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 183 ms; skipped=0; WAL=hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134933206, size=195.8 K, length=200484, corrupted=false, cancelled=false 2024-11-20T20:35:33,916 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134933206, journal: Splitting hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134933206, size=195.8 K (200484bytes) at 1732134933715Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000003001-wal.1732134933206.temp at 1732134933755 (+40 ms)Split 1024 edits, skipped 0 edits. at 1732134933762 (+7 ms)Split 2048 edits, skipped 0 edits. at 1732134933782 (+20 ms)Finishing writing output for hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134933206 so closing down at 1732134933807 (+25 ms)3 split writer threads finished at 1732134933810 (+3 ms)Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000003001-wal.1732134933206.temp (wrote 3000 edits, skipped 0 edits in 20 ms) at 1732134933912 (+102 ms)Rename recovered edits hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000003001-wal.1732134933206.temp to hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000006000 at 1732134933915 (+3 ms)Processed 3000 edits across 1 Regions in 183 ms; skipped=0; WAL=hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134933206, size=195.8 K, length=200484, corrupted=false, cancelled=false at 1732134933916 (+1 ms) 2024-11-20T20:35:33,926 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134933206 to hdfs://localhost:33605/hbase/oldWALs/wal.1732134933206 2024-11-20T20:35:33,928 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000006000 2024-11-20T20:35:33,928 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T20:35:33,932 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/test2727-manual,16010,1732134931957, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:33,953 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134933932, exclude list is [], retry=0 2024-11-20T20:35:33,959 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:33,959 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:33,960 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:33,966 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732134931957/wal.1732134933932 2024-11-20T20:35:33,967 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:39759:39759),(127.0.0.1/127.0.0.1:38233:38233)] 2024-11-20T20:35:33,967 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => bfa90791c9bfd148422bed44d5ff1248, NAME => 'test2727,,1732134931961.bfa90791c9bfd148422bed44d5ff1248.', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:35:33,967 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732134931961.bfa90791c9bfd148422bed44d5ff1248.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:33,967 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for bfa90791c9bfd148422bed44d5ff1248 2024-11-20T20:35:33,967 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for bfa90791c9bfd148422bed44d5ff1248 2024-11-20T20:35:33,970 INFO [StoreOpener-bfa90791c9bfd148422bed44d5ff1248-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bfa90791c9bfd148422bed44d5ff1248 2024-11-20T20:35:33,971 INFO [StoreOpener-bfa90791c9bfd148422bed44d5ff1248-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa90791c9bfd148422bed44d5ff1248 columnFamilyName a 2024-11-20T20:35:33,971 DEBUG [StoreOpener-bfa90791c9bfd148422bed44d5ff1248-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:33,973 INFO [StoreOpener-bfa90791c9bfd148422bed44d5ff1248-1 {}] regionserver.HStore(327): Store=bfa90791c9bfd148422bed44d5ff1248/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:33,973 INFO [StoreOpener-bfa90791c9bfd148422bed44d5ff1248-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bfa90791c9bfd148422bed44d5ff1248 2024-11-20T20:35:33,974 INFO [StoreOpener-bfa90791c9bfd148422bed44d5ff1248-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa90791c9bfd148422bed44d5ff1248 columnFamilyName b 2024-11-20T20:35:33,975 DEBUG [StoreOpener-bfa90791c9bfd148422bed44d5ff1248-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:33,976 INFO [StoreOpener-bfa90791c9bfd148422bed44d5ff1248-1 {}] regionserver.HStore(327): Store=bfa90791c9bfd148422bed44d5ff1248/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:33,976 INFO [StoreOpener-bfa90791c9bfd148422bed44d5ff1248-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bfa90791c9bfd148422bed44d5ff1248 2024-11-20T20:35:33,981 INFO [StoreOpener-bfa90791c9bfd148422bed44d5ff1248-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa90791c9bfd148422bed44d5ff1248 columnFamilyName c 2024-11-20T20:35:33,981 DEBUG [StoreOpener-bfa90791c9bfd148422bed44d5ff1248-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:33,989 INFO [StoreOpener-bfa90791c9bfd148422bed44d5ff1248-1 {}] regionserver.HStore(327): Store=bfa90791c9bfd148422bed44d5ff1248/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:33,989 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bfa90791c9bfd148422bed44d5ff1248 2024-11-20T20:35:33,992 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248 2024-11-20T20:35:33,996 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248 2024-11-20T20:35:33,998 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000003000 2024-11-20T20:35:34,004 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:34,084 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000003000 2024-11-20T20:35:34,096 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000006000 2024-11-20T20:35:34,100 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:34,184 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000006000 2024-11-20T20:35:34,184 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bfa90791c9bfd148422bed44d5ff1248 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-20T20:35:34,223 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/.tmp/a/69ef9537a13348f38353d76458d320e8 is 41, key is test2727/a:100/1732134933355/Put/seqid=0 2024-11-20T20:35:34,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741871_1048 (size=84227) 2024-11-20T20:35:34,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741871_1048 (size=84227) 2024-11-20T20:35:34,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741871_1048 (size=84227) 2024-11-20T20:35:34,270 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/.tmp/a/69ef9537a13348f38353d76458d320e8 2024-11-20T20:35:34,327 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/.tmp/b/9800088c44be487e8ceca1883d70147d is 41, key is test2727/b:100/1732134933538/Put/seqid=0 2024-11-20T20:35:34,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741872_1049 (size=84609) 2024-11-20T20:35:34,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741872_1049 (size=84609) 2024-11-20T20:35:34,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741872_1049 (size=84609) 2024-11-20T20:35:34,408 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/.tmp/b/9800088c44be487e8ceca1883d70147d 2024-11-20T20:35:34,498 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/.tmp/c/92b32872e47d457588e0eea92c2b568c is 41, key is test2727/c:100/1732134933615/Put/seqid=0 2024-11-20T20:35:34,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741873_1050 (size=84609) 2024-11-20T20:35:34,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741873_1050 (size=84609) 2024-11-20T20:35:34,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741873_1050 (size=84609) 2024-11-20T20:35:34,551 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/.tmp/c/92b32872e47d457588e0eea92c2b568c 2024-11-20T20:35:34,583 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/.tmp/a/69ef9537a13348f38353d76458d320e8 as hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/a/69ef9537a13348f38353d76458d320e8 2024-11-20T20:35:34,601 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/a/69ef9537a13348f38353d76458d320e8, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-20T20:35:34,605 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/.tmp/b/9800088c44be487e8ceca1883d70147d as hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/b/9800088c44be487e8ceca1883d70147d 2024-11-20T20:35:34,620 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/b/9800088c44be487e8ceca1883d70147d, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-20T20:35:34,622 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/.tmp/c/92b32872e47d457588e0eea92c2b568c as hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/c/92b32872e47d457588e0eea92c2b568c 2024-11-20T20:35:34,640 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/c/92b32872e47d457588e0eea92c2b568c, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-20T20:35:34,640 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for bfa90791c9bfd148422bed44d5ff1248 in 456ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-20T20:35:34,642 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000003000 2024-11-20T20:35:34,642 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/0000000000000006000 2024-11-20T20:35:34,644 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bfa90791c9bfd148422bed44d5ff1248 2024-11-20T20:35:34,644 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bfa90791c9bfd148422bed44d5ff1248 2024-11-20T20:35:34,645 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T20:35:34,647 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bfa90791c9bfd148422bed44d5ff1248 2024-11-20T20:35:34,650 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/hbase/data/default/test2727/bfa90791c9bfd148422bed44d5ff1248/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-20T20:35:34,651 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bfa90791c9bfd148422bed44d5ff1248; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58793204, jitterRate=-0.1239129900932312}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T20:35:34,653 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bfa90791c9bfd148422bed44d5ff1248: Writing region info on filesystem at 1732134933968Initializing all the Stores at 1732134933969 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134933969Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134933970 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134933970Obtaining lock to block concurrent updates at 1732134934185 (+215 ms)Preparing flush snapshotting stores in bfa90791c9bfd148422bed44d5ff1248 at 1732134934185Finished memstore snapshotting test2727,,1732134931961.bfa90791c9bfd148422bed44d5ff1248., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732134934185Flushing stores of test2727,,1732134931961.bfa90791c9bfd148422bed44d5ff1248. at 1732134934185Flushing bfa90791c9bfd148422bed44d5ff1248/a: creating writer at 1732134934185Flushing bfa90791c9bfd148422bed44d5ff1248/a: appending metadata at 1732134934222 (+37 ms)Flushing bfa90791c9bfd148422bed44d5ff1248/a: closing flushed file at 1732134934223 (+1 ms)Flushing bfa90791c9bfd148422bed44d5ff1248/b: creating writer at 1732134934282 (+59 ms)Flushing bfa90791c9bfd148422bed44d5ff1248/b: appending metadata at 1732134934325 (+43 ms)Flushing bfa90791c9bfd148422bed44d5ff1248/b: closing flushed file at 1732134934325Flushing bfa90791c9bfd148422bed44d5ff1248/c: creating writer at 1732134934436 (+111 ms)Flushing bfa90791c9bfd148422bed44d5ff1248/c: appending metadata at 1732134934497 (+61 ms)Flushing bfa90791c9bfd148422bed44d5ff1248/c: closing flushed file at 1732134934497Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43ef1367: reopening flushed file at 1732134934575 (+78 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@531372ce: reopening flushed file at 1732134934601 (+26 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c700255: reopening flushed file at 1732134934621 (+20 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for bfa90791c9bfd148422bed44d5ff1248 in 456ms, sequenceid=6000, compaction requested=false; wal=null at 1732134934641 (+20 ms)Cleaning up temporary data from old regions at 1732134934644 (+3 ms)Region opened successfully at 1732134934653 (+9 ms) 2024-11-20T20:35:34,655 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-20T20:35:34,655 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bfa90791c9bfd148422bed44d5ff1248, disabling compactions & flushes 2024-11-20T20:35:34,655 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732134931961.bfa90791c9bfd148422bed44d5ff1248. 2024-11-20T20:35:34,655 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732134931961.bfa90791c9bfd148422bed44d5ff1248. 2024-11-20T20:35:34,656 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732134931961.bfa90791c9bfd148422bed44d5ff1248. after waiting 0 ms 2024-11-20T20:35:34,656 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732134931961.bfa90791c9bfd148422bed44d5ff1248. 2024-11-20T20:35:34,663 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732134931961.bfa90791c9bfd148422bed44d5ff1248. 2024-11-20T20:35:34,663 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bfa90791c9bfd148422bed44d5ff1248: Waiting for close lock at 1732134934655Disabling compacts and flushes for region at 1732134934655Disabling writes for close at 1732134934656 (+1 ms)Writing region close event to WAL at 1732134934663 (+7 ms)Closed at 1732134934663 2024-11-20T20:35:34,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741870_1047 (size=93) 2024-11-20T20:35:34,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741870_1047 (size=93) 2024-11-20T20:35:34,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741870_1047 (size=93) 2024-11-20T20:35:34,692 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T20:35:34,692 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732134933932) 2024-11-20T20:35:34,720 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=397 (was 400), OpenFileDescriptor=921 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=933 (was 933), ProcessCount=11 (was 11), AvailableMemoryMB=1998 (was 2246) 2024-11-20T20:35:34,749 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=397, OpenFileDescriptor=921, MaxFileDescriptor=1048576, SystemLoadAverage=933, ProcessCount=11, AvailableMemoryMB=1995 2024-11-20T20:35:34,774 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:34,794 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732134934773, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:34,795 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732134934795 2024-11-20T20:35:34,828 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732134934773/wal.1732134934795 2024-11-20T20:35:34,857 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:39759:39759),(127.0.0.1/127.0.0.1:38233:38233)] 2024-11-20T20:35:34,873 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 361e172eef73f99f5b8e52107baf5d1b, NAME => 'testSequentialEditLogSeqNum,,1732134934775.361e172eef73f99f5b8e52107baf5d1b.', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:35:34,874 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732134934775.361e172eef73f99f5b8e52107baf5d1b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:34,874 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 361e172eef73f99f5b8e52107baf5d1b 2024-11-20T20:35:34,874 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 361e172eef73f99f5b8e52107baf5d1b 2024-11-20T20:35:34,879 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:33605/hbase/data/default/testSequentialEditLogSeqNum/361e172eef73f99f5b8e52107baf5d1b doesn't exist for region: 361e172eef73f99f5b8e52107baf5d1b on table testSequentialEditLogSeqNum 2024-11-20T20:35:34,889 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 361e172eef73f99f5b8e52107baf5d1b on table testSequentialEditLogSeqNum 2024-11-20T20:35:35,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741875_1052 (size=62) 2024-11-20T20:35:35,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741875_1052 (size=62) 2024-11-20T20:35:35,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741875_1052 (size=62) 2024-11-20T20:35:35,032 INFO [StoreOpener-361e172eef73f99f5b8e52107baf5d1b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 361e172eef73f99f5b8e52107baf5d1b 2024-11-20T20:35:35,039 INFO [StoreOpener-361e172eef73f99f5b8e52107baf5d1b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 361e172eef73f99f5b8e52107baf5d1b columnFamilyName a 2024-11-20T20:35:35,039 DEBUG [StoreOpener-361e172eef73f99f5b8e52107baf5d1b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:35,040 INFO [StoreOpener-361e172eef73f99f5b8e52107baf5d1b-1 {}] regionserver.HStore(327): Store=361e172eef73f99f5b8e52107baf5d1b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:35,040 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 361e172eef73f99f5b8e52107baf5d1b 2024-11-20T20:35:35,045 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testSequentialEditLogSeqNum/361e172eef73f99f5b8e52107baf5d1b 2024-11-20T20:35:35,047 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testSequentialEditLogSeqNum/361e172eef73f99f5b8e52107baf5d1b 2024-11-20T20:35:35,048 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 361e172eef73f99f5b8e52107baf5d1b 2024-11-20T20:35:35,048 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 361e172eef73f99f5b8e52107baf5d1b 2024-11-20T20:35:35,051 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 361e172eef73f99f5b8e52107baf5d1b 2024-11-20T20:35:35,064 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/hbase/data/default/testSequentialEditLogSeqNum/361e172eef73f99f5b8e52107baf5d1b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T20:35:35,070 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 361e172eef73f99f5b8e52107baf5d1b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62574510, jitterRate=-0.0675671398639679}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T20:35:35,072 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 361e172eef73f99f5b8e52107baf5d1b: Writing region info on filesystem at 1732134934874Initializing all the Stores at 1732134935030 (+156 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134935030Cleaning up temporary data from old regions at 1732134935048 (+18 ms)Region opened successfully at 1732134935071 (+23 ms) 2024-11-20T20:35:35,137 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 361e172eef73f99f5b8e52107baf5d1b 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-20T20:35:35,194 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testSequentialEditLogSeqNum/361e172eef73f99f5b8e52107baf5d1b/.tmp/a/29af200bc5994417a00c497fc6b0fa54 is 81, key is testSequentialEditLogSeqNum/a:x0/1732134935072/Put/seqid=0 2024-11-20T20:35:35,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741876_1053 (size=5833) 2024-11-20T20:35:35,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741876_1053 (size=5833) 2024-11-20T20:35:35,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741876_1053 (size=5833) 2024-11-20T20:35:35,278 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/testSequentialEditLogSeqNum/361e172eef73f99f5b8e52107baf5d1b/.tmp/a/29af200bc5994417a00c497fc6b0fa54 2024-11-20T20:35:35,333 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testSequentialEditLogSeqNum/361e172eef73f99f5b8e52107baf5d1b/.tmp/a/29af200bc5994417a00c497fc6b0fa54 as hdfs://localhost:33605/hbase/data/default/testSequentialEditLogSeqNum/361e172eef73f99f5b8e52107baf5d1b/a/29af200bc5994417a00c497fc6b0fa54 2024-11-20T20:35:35,392 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/testSequentialEditLogSeqNum/361e172eef73f99f5b8e52107baf5d1b/a/29af200bc5994417a00c497fc6b0fa54, entries=10, sequenceid=13, filesize=5.7 K 2024-11-20T20:35:35,399 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 361e172eef73f99f5b8e52107baf5d1b in 262ms, sequenceid=13, compaction requested=false 2024-11-20T20:35:35,400 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 361e172eef73f99f5b8e52107baf5d1b: 2024-11-20T20:35:35,510 INFO [FSHLog-0-hdfs://localhost:33605/hbase-prefix:default {}] wal.AbstractFSWAL(1368): Slow sync cost: 109 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK], DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK], DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK]] 2024-11-20T20:35:35,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T20:35:35,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T20:35:35,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T20:35:35,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T20:35:35,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T20:35:35,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741874_1051 (size=1616) 2024-11-20T20:35:35,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741874_1051 (size=1616) 2024-11-20T20:35:35,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741874_1051 (size=1616) 2024-11-20T20:35:35,613 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33605/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732134934773/wal.1732134934795, size=1.6 K (1616bytes) 2024-11-20T20:35:35,613 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33605/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732134934773/wal.1732134934795 2024-11-20T20:35:35,614 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33605/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732134934773/wal.1732134934795 after 1ms 2024-11-20T20:35:35,618 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732134934773/wal.1732134934795: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:35,618 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33605/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732134934773/wal.1732134934795 took 6ms 2024-11-20T20:35:35,623 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33605/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732134934773/wal.1732134934795 so closing down 2024-11-20T20:35:35,623 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T20:35:35,626 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732134934795.temp 2024-11-20T20:35:35,632 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testSequentialEditLogSeqNum/361e172eef73f99f5b8e52107baf5d1b/recovered.edits/0000000000000000003-wal.1732134934795.temp 2024-11-20T20:35:35,632 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T20:35:35,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741877_1054 (size=1175) 2024-11-20T20:35:35,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741877_1054 (size=1175) 2024-11-20T20:35:35,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741877_1054 (size=1175) 2024-11-20T20:35:35,655 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testSequentialEditLogSeqNum/361e172eef73f99f5b8e52107baf5d1b/recovered.edits/0000000000000000003-wal.1732134934795.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-20T20:35:35,657 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33605/hbase/data/default/testSequentialEditLogSeqNum/361e172eef73f99f5b8e52107baf5d1b/recovered.edits/0000000000000000003-wal.1732134934795.temp to hdfs://localhost:33605/hbase/data/default/testSequentialEditLogSeqNum/361e172eef73f99f5b8e52107baf5d1b/recovered.edits/0000000000000000020 2024-11-20T20:35:35,657 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 37 ms; skipped=2; WAL=hdfs://localhost:33605/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732134934773/wal.1732134934795, size=1.6 K, length=1616, corrupted=false, cancelled=false 2024-11-20T20:35:35,657 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33605/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732134934773/wal.1732134934795, journal: Splitting hdfs://localhost:33605/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732134934773/wal.1732134934795, size=1.6 K (1616bytes) at 1732134935613Finishing writing output for hdfs://localhost:33605/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732134934773/wal.1732134934795 so closing down at 1732134935623 (+10 ms)Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testSequentialEditLogSeqNum/361e172eef73f99f5b8e52107baf5d1b/recovered.edits/0000000000000000003-wal.1732134934795.temp at 1732134935632 (+9 ms)3 split writer threads finished at 1732134935632Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testSequentialEditLogSeqNum/361e172eef73f99f5b8e52107baf5d1b/recovered.edits/0000000000000000003-wal.1732134934795.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732134935655 (+23 ms)Rename recovered edits hdfs://localhost:33605/hbase/data/default/testSequentialEditLogSeqNum/361e172eef73f99f5b8e52107baf5d1b/recovered.edits/0000000000000000003-wal.1732134934795.temp to hdfs://localhost:33605/hbase/data/default/testSequentialEditLogSeqNum/361e172eef73f99f5b8e52107baf5d1b/recovered.edits/0000000000000000020 at 1732134935657 (+2 ms)Processed 17 edits across 1 Regions in 37 ms; skipped=2; WAL=hdfs://localhost:33605/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732134934773/wal.1732134934795, size=1.6 K, length=1616, corrupted=false, cancelled=false at 1732134935657 2024-11-20T20:35:35,677 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=402 (was 397) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:45354 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:59312 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=957 (was 921) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=933 (was 933), ProcessCount=11 (was 11), AvailableMemoryMB=1781 (was 1995) 2024-11-20T20:35:35,690 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=402, OpenFileDescriptor=957, MaxFileDescriptor=1048576, SystemLoadAverage=933, ProcessCount=11, AvailableMemoryMB=1780 2024-11-20T20:35:35,709 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:35,711 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:35,737 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T20:35:35,748 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-93617854, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/hregion-93617854, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:35,768 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-93617854/hregion-93617854.1732134935749, exclude list is [], retry=0 2024-11-20T20:35:35,775 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:35,777 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:35,780 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:35,801 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-93617854/hregion-93617854.1732134935749 2024-11-20T20:35:35,804 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:39759:39759),(127.0.0.1/127.0.0.1:38233:38233)] 2024-11-20T20:35:35,805 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6fef7295820879883e79289e0a1ea01e, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732134935710.6fef7295820879883e79289e0a1ea01e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33605/hbase 2024-11-20T20:35:35,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741879_1056 (size=70) 2024-11-20T20:35:35,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741879_1056 (size=70) 2024-11-20T20:35:35,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741879_1056 (size=70) 2024-11-20T20:35:35,840 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732134935710.6fef7295820879883e79289e0a1ea01e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:35,842 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:35,845 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fef7295820879883e79289e0a1ea01e columnFamilyName a 2024-11-20T20:35:35,845 DEBUG [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:35,846 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] regionserver.HStore(327): Store=6fef7295820879883e79289e0a1ea01e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:35,846 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:35,848 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fef7295820879883e79289e0a1ea01e columnFamilyName b 2024-11-20T20:35:35,848 DEBUG [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:35,850 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] regionserver.HStore(327): Store=6fef7295820879883e79289e0a1ea01e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:35,851 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:35,858 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fef7295820879883e79289e0a1ea01e columnFamilyName c 2024-11-20T20:35:35,858 DEBUG [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:35,859 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] regionserver.HStore(327): Store=6fef7295820879883e79289e0a1ea01e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:35,859 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:35,860 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:35,861 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:35,863 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:35,863 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:35,864 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T20:35:35,866 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:35,869 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T20:35:35,869 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6fef7295820879883e79289e0a1ea01e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72227377, jitterRate=0.07627178728580475}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T20:35:35,870 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6fef7295820879883e79289e0a1ea01e: Writing region info on filesystem at 1732134935840Initializing all the Stores at 1732134935841 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134935841Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134935842 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134935842Cleaning up temporary data from old regions at 1732134935863 (+21 ms)Region opened successfully at 1732134935870 (+7 ms) 2024-11-20T20:35:35,870 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6fef7295820879883e79289e0a1ea01e, disabling compactions & flushes 2024-11-20T20:35:35,871 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732134935710.6fef7295820879883e79289e0a1ea01e. 2024-11-20T20:35:35,871 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732134935710.6fef7295820879883e79289e0a1ea01e. 2024-11-20T20:35:35,871 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732134935710.6fef7295820879883e79289e0a1ea01e. after waiting 0 ms 2024-11-20T20:35:35,871 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732134935710.6fef7295820879883e79289e0a1ea01e. 2024-11-20T20:35:35,877 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732134935710.6fef7295820879883e79289e0a1ea01e. 2024-11-20T20:35:35,877 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6fef7295820879883e79289e0a1ea01e: Waiting for close lock at 1732134935870Disabling compacts and flushes for region at 1732134935870Disabling writes for close at 1732134935871 (+1 ms)Writing region close event to WAL at 1732134935876 (+5 ms)Closed at 1732134935876 2024-11-20T20:35:35,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741878_1055 (size=93) 2024-11-20T20:35:35,885 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-93617854/hregion-93617854.1732134935749 not finished, retry = 0 2024-11-20T20:35:35,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741878_1055 (size=93) 2024-11-20T20:35:35,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741878_1055 (size=93) 2024-11-20T20:35:36,008 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T20:35:36,008 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-93617854:(num 1732134935749) 2024-11-20T20:35:36,008 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T20:35:36,011 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732134935707, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:36,031 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732134935707/wal.1732134936012, exclude list is [], retry=0 2024-11-20T20:35:36,037 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:36,038 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:36,038 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:36,046 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732134935707/wal.1732134936012 2024-11-20T20:35:36,047 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39759:39759),(127.0.0.1/127.0.0.1:38233:38233),(127.0.0.1/127.0.0.1:46511:46511)] 2024-11-20T20:35:36,047 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6fef7295820879883e79289e0a1ea01e, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732134935710.6fef7295820879883e79289e0a1ea01e.', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:35:36,047 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732134935710.6fef7295820879883e79289e0a1ea01e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:36,047 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:36,048 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:36,049 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:36,051 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fef7295820879883e79289e0a1ea01e columnFamilyName a 2024-11-20T20:35:36,051 DEBUG [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:36,051 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] regionserver.HStore(327): Store=6fef7295820879883e79289e0a1ea01e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:36,052 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:36,053 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fef7295820879883e79289e0a1ea01e columnFamilyName b 2024-11-20T20:35:36,053 DEBUG [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:36,054 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] regionserver.HStore(327): Store=6fef7295820879883e79289e0a1ea01e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:36,054 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:36,055 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fef7295820879883e79289e0a1ea01e columnFamilyName c 2024-11-20T20:35:36,055 DEBUG [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:36,056 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] regionserver.HStore(327): Store=6fef7295820879883e79289e0a1ea01e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:36,056 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:36,057 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:36,058 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:36,059 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:36,059 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:36,060 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T20:35:36,062 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:36,063 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6fef7295820879883e79289e0a1ea01e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61912149, jitterRate=-0.07743708789348602}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T20:35:36,063 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6fef7295820879883e79289e0a1ea01e: Writing region info on filesystem at 1732134936048Initializing all the Stores at 1732134936049 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134936049Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134936049Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134936049Cleaning up temporary data from old regions at 1732134936060 (+11 ms)Region opened successfully at 1732134936063 (+3 ms) 2024-11-20T20:35:36,068 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732134936067/Put/seqid=0 2024-11-20T20:35:36,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741881_1058 (size=4826) 2024-11-20T20:35:36,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741881_1058 (size=4826) 2024-11-20T20:35:36,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741881_1058 (size=4826) 2024-11-20T20:35:36,085 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33605/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 6fef7295820879883e79289e0a1ea01e/a 2024-11-20T20:35:36,095 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-20T20:35:36,095 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-20T20:35:36,096 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6fef7295820879883e79289e0a1ea01e: 2024-11-20T20:35:36,098 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e/a/17c66e05374243e2ad6416b5537c6074_SeqId_3_ 2024-11-20T20:35:36,100 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33605/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 6fef7295820879883e79289e0a1ea01e/a as hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e/a/17c66e05374243e2ad6416b5537c6074_SeqId_3_ - updating store file list. 2024-11-20T20:35:36,109 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 17c66e05374243e2ad6416b5537c6074_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-20T20:35:36,109 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e/a/17c66e05374243e2ad6416b5537c6074_SeqId_3_ into 6fef7295820879883e79289e0a1ea01e/a 2024-11-20T20:35:36,110 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33605/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 6fef7295820879883e79289e0a1ea01e/a (new location: hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e/a/17c66e05374243e2ad6416b5537c6074_SeqId_3_) 2024-11-20T20:35:36,162 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33605/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732134935707/wal.1732134936012, size=0 (0bytes) 2024-11-20T20:35:36,162 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33605/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732134935707/wal.1732134936012 might be still open, length is 0 2024-11-20T20:35:36,162 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33605/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732134935707/wal.1732134936012 2024-11-20T20:35:36,163 WARN [IPC Server handler 4 on default port 33605 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732134935707/wal.1732134936012 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-20T20:35:36,163 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33605/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732134935707/wal.1732134936012 after 1ms 2024-11-20T20:35:38,011 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:45466 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:40323:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45466 dst: /127.0.0.1:40323 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:40323 remote=/127.0.0.1:45466]. Total timeout mills is 60000, 58108 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:35:38,012 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:59414 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:44519:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59414 dst: /127.0.0.1:44519 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:35:38,012 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:60342 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:35033:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60342 dst: /127.0.0.1:35033 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:35:38,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741880_1059 (size=434) 2024-11-20T20:35:38,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741880_1059 (size=434) 2024-11-20T20:35:38,415 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-20T20:35:38,416 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-20T20:35:38,417 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-20T20:35:38,417 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-20T20:35:40,164 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33605/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732134935707/wal.1732134936012 after 4002ms 2024-11-20T20:35:40,169 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732134935707/wal.1732134936012: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:40,169 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33605/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732134935707/wal.1732134936012 took 4007ms 2024-11-20T20:35:40,173 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33605/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732134935707/wal.1732134936012; continuing. 2024-11-20T20:35:40,173 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33605/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732134935707/wal.1732134936012 so closing down 2024-11-20T20:35:40,173 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T20:35:40,176 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732134936012.temp 2024-11-20T20:35:40,178 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e/recovered.edits/0000000000000000005-wal.1732134936012.temp 2024-11-20T20:35:40,178 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T20:35:40,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741882_1060 (size=236) 2024-11-20T20:35:40,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741882_1060 (size=236) 2024-11-20T20:35:40,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741882_1060 (size=236) 2024-11-20T20:35:40,187 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e/recovered.edits/0000000000000000005-wal.1732134936012.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-20T20:35:40,189 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e/recovered.edits/0000000000000000005-wal.1732134936012.temp to hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e/recovered.edits/0000000000000000005 2024-11-20T20:35:40,189 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 19 ms; skipped=1; WAL=hdfs://localhost:33605/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732134935707/wal.1732134936012, size=0, length=0, corrupted=false, cancelled=false 2024-11-20T20:35:40,189 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33605/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732134935707/wal.1732134936012, journal: Splitting hdfs://localhost:33605/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732134935707/wal.1732134936012, size=0 (0bytes) at 1732134936162Finishing writing output for hdfs://localhost:33605/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732134935707/wal.1732134936012 so closing down at 1732134940173 (+4011 ms)Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e/recovered.edits/0000000000000000005-wal.1732134936012.temp at 1732134940178 (+5 ms)3 split writer threads finished at 1732134940178Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e/recovered.edits/0000000000000000005-wal.1732134936012.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732134940187 (+9 ms)Rename recovered edits hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e/recovered.edits/0000000000000000005-wal.1732134936012.temp to hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e/recovered.edits/0000000000000000005 at 1732134940189 (+2 ms)Processed 2 edits across 1 Regions in 19 ms; skipped=1; WAL=hdfs://localhost:33605/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732134935707/wal.1732134936012, size=0, length=0, corrupted=false, cancelled=false at 1732134940189 2024-11-20T20:35:40,191 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33605/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732134935707/wal.1732134936012 to hdfs://localhost:33605/hbase/oldWALs/wal.1732134936012 2024-11-20T20:35:40,192 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e/recovered.edits/0000000000000000005 2024-11-20T20:35:40,193 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T20:35:40,195 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732134935707, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:40,208 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732134935707/wal.1732134940195, exclude list is [], retry=0 2024-11-20T20:35:40,211 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:40,212 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:40,212 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:40,214 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732134935707/wal.1732134940195 2024-11-20T20:35:40,215 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:39759:39759),(127.0.0.1/127.0.0.1:38233:38233)] 2024-11-20T20:35:40,215 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6fef7295820879883e79289e0a1ea01e, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732134935710.6fef7295820879883e79289e0a1ea01e.', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:35:40,216 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732134935710.6fef7295820879883e79289e0a1ea01e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:40,216 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:40,216 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:40,218 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:40,219 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fef7295820879883e79289e0a1ea01e columnFamilyName a 2024-11-20T20:35:40,219 DEBUG [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:40,226 DEBUG [StoreFileOpener-6fef7295820879883e79289e0a1ea01e-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 17c66e05374243e2ad6416b5537c6074_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-20T20:35:40,226 DEBUG [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e/a/17c66e05374243e2ad6416b5537c6074_SeqId_3_ 2024-11-20T20:35:40,226 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] regionserver.HStore(327): Store=6fef7295820879883e79289e0a1ea01e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:40,226 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:40,228 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fef7295820879883e79289e0a1ea01e columnFamilyName b 2024-11-20T20:35:40,228 DEBUG [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:40,228 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] regionserver.HStore(327): Store=6fef7295820879883e79289e0a1ea01e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:40,229 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:40,230 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fef7295820879883e79289e0a1ea01e columnFamilyName c 2024-11-20T20:35:40,230 DEBUG [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:40,231 INFO [StoreOpener-6fef7295820879883e79289e0a1ea01e-1 {}] regionserver.HStore(327): Store=6fef7295820879883e79289e0a1ea01e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:40,231 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:40,232 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:40,233 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:40,234 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e/recovered.edits/0000000000000000005 2024-11-20T20:35:40,236 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:40,237 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e/recovered.edits/0000000000000000005 2024-11-20T20:35:40,237 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6fef7295820879883e79289e0a1ea01e 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-20T20:35:40,252 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e/.tmp/a/13862aaecda1445aa954a1f243471437 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732134936117/Put/seqid=0 2024-11-20T20:35:40,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741884_1062 (size=5149) 2024-11-20T20:35:40,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741884_1062 (size=5149) 2024-11-20T20:35:40,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741884_1062 (size=5149) 2024-11-20T20:35:40,260 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e/.tmp/a/13862aaecda1445aa954a1f243471437 2024-11-20T20:35:40,266 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e/.tmp/a/13862aaecda1445aa954a1f243471437 as hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e/a/13862aaecda1445aa954a1f243471437 2024-11-20T20:35:40,272 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e/a/13862aaecda1445aa954a1f243471437, entries=1, sequenceid=5, filesize=5.0 K 2024-11-20T20:35:40,273 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 6fef7295820879883e79289e0a1ea01e in 36ms, sequenceid=5, compaction requested=false; wal=null 2024-11-20T20:35:40,273 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e/recovered.edits/0000000000000000005 2024-11-20T20:35:40,276 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:40,276 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:40,277 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T20:35:40,279 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6fef7295820879883e79289e0a1ea01e 2024-11-20T20:35:40,282 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/6fef7295820879883e79289e0a1ea01e/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-20T20:35:40,283 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6fef7295820879883e79289e0a1ea01e; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61423345, jitterRate=-0.08472083508968353}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T20:35:40,283 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6fef7295820879883e79289e0a1ea01e: Writing region info on filesystem at 1732134940216Initializing all the Stores at 1732134940217 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134940217Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134940217Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134940217Obtaining lock to block concurrent updates at 1732134940237 (+20 ms)Preparing flush snapshotting stores in 6fef7295820879883e79289e0a1ea01e at 1732134940237Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732134935710.6fef7295820879883e79289e0a1ea01e., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732134940237Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732134935710.6fef7295820879883e79289e0a1ea01e. at 1732134940237Flushing 6fef7295820879883e79289e0a1ea01e/a: creating writer at 1732134940238 (+1 ms)Flushing 6fef7295820879883e79289e0a1ea01e/a: appending metadata at 1732134940251 (+13 ms)Flushing 6fef7295820879883e79289e0a1ea01e/a: closing flushed file at 1732134940251Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@97a2406: reopening flushed file at 1732134940265 (+14 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 6fef7295820879883e79289e0a1ea01e in 36ms, sequenceid=5, compaction requested=false; wal=null at 1732134940273 (+8 ms)Cleaning up temporary data from old regions at 1732134940276 (+3 ms)Region opened successfully at 1732134940283 (+7 ms) 2024-11-20T20:35:40,288 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6fef7295820879883e79289e0a1ea01e, disabling compactions & flushes 2024-11-20T20:35:40,288 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732134935710.6fef7295820879883e79289e0a1ea01e. 2024-11-20T20:35:40,288 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732134935710.6fef7295820879883e79289e0a1ea01e. 2024-11-20T20:35:40,288 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732134935710.6fef7295820879883e79289e0a1ea01e. after waiting 0 ms 2024-11-20T20:35:40,288 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732134935710.6fef7295820879883e79289e0a1ea01e. 2024-11-20T20:35:40,289 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732134935710.6fef7295820879883e79289e0a1ea01e. 2024-11-20T20:35:40,289 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6fef7295820879883e79289e0a1ea01e: Waiting for close lock at 1732134940288Disabling compacts and flushes for region at 1732134940288Disabling writes for close at 1732134940288Writing region close event to WAL at 1732134940289 (+1 ms)Closed at 1732134940289 2024-11-20T20:35:40,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741883_1061 (size=93) 2024-11-20T20:35:40,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741883_1061 (size=93) 2024-11-20T20:35:40,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741883_1061 (size=93) 2024-11-20T20:35:40,297 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T20:35:40,297 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732134940195) 2024-11-20T20:35:40,310 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=411 (was 402) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:33605 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_634554242_22 at /127.0.0.1:60378 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:33605 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_634554242_22 at /127.0.0.1:59470 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_634554242_22 at /127.0.0.1:45480 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 957) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=922 (was 933), ProcessCount=11 (was 11), AvailableMemoryMB=1729 (was 1780) 2024-11-20T20:35:40,323 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=411, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=922, ProcessCount=11, AvailableMemoryMB=1729 2024-11-20T20:35:40,344 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:40,350 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T20:35:40,355 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is d7d59561a64e,46641,1732134918030 2024-11-20T20:35:40,358 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@69f7cdbc 2024-11-20T20:35:40,359 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T20:35:40,361 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T20:35:40,365 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46641 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T20:35:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46641 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-20T20:35:40,374 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T20:35:40,376 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46641 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-20T20:35:40,377 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:40,379 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T20:35:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46641 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T20:35:40,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741885_1063 (size=694) 2024-11-20T20:35:40,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741885_1063 (size=694) 2024-11-20T20:35:40,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741885_1063 (size=694) 2024-11-20T20:35:40,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46641 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T20:35:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46641 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T20:35:40,796 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e84bafb3668f028684842276e5650d4c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6 2024-11-20T20:35:40,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741886_1064 (size=77) 2024-11-20T20:35:40,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741886_1064 (size=77) 2024-11-20T20:35:40,820 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:40,820 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing e84bafb3668f028684842276e5650d4c, disabling compactions & flushes 2024-11-20T20:35:40,820 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:40,820 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:40,820 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. after waiting 0 ms 2024-11-20T20:35:40,820 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:40,820 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:40,821 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for e84bafb3668f028684842276e5650d4c: Waiting for close lock at 1732134940820Disabling compacts and flushes for region at 1732134940820Disabling writes for close at 1732134940820Writing region close event to WAL at 1732134940820Closed at 1732134940820 2024-11-20T20:35:40,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741886_1064 (size=77) 2024-11-20T20:35:40,823 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T20:35:40,829 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732134940823"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732134940823"}]},"ts":"1732134940823"} 2024-11-20T20:35:40,841 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T20:35:40,845 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T20:35:40,848 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732134940845"}]},"ts":"1732134940845"} 2024-11-20T20:35:40,864 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-20T20:35:40,865 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {d7d59561a64e=0} racks are {/default-rack=0} 2024-11-20T20:35:40,868 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T20:35:40,868 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T20:35:40,868 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-20T20:35:40,868 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T20:35:40,868 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T20:35:40,868 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-20T20:35:40,868 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T20:35:40,868 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T20:35:40,868 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-20T20:35:40,868 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T20:35:40,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e84bafb3668f028684842276e5650d4c, ASSIGN}] 2024-11-20T20:35:40,880 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e84bafb3668f028684842276e5650d4c, ASSIGN 2024-11-20T20:35:40,882 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e84bafb3668f028684842276e5650d4c, ASSIGN; state=OFFLINE, location=d7d59561a64e,36639,1732134919021; forceNewPlan=false, retain=false 2024-11-20T20:35:41,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46641 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T20:35:41,036 INFO [d7d59561a64e:46641 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T20:35:41,041 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e84bafb3668f028684842276e5650d4c, regionState=OPENING, regionLocation=d7d59561a64e,36639,1732134919021 2024-11-20T20:35:41,045 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e84bafb3668f028684842276e5650d4c, ASSIGN because future has completed 2024-11-20T20:35:41,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e84bafb3668f028684842276e5650d4c, server=d7d59561a64e,36639,1732134919021}] 2024-11-20T20:35:41,222 INFO [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:41,222 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e84bafb3668f028684842276e5650d4c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c.', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:35:41,223 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:41,223 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:41,223 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:41,223 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:41,226 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:41,228 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e84bafb3668f028684842276e5650d4c columnFamilyName cf1 2024-11-20T20:35:41,228 DEBUG [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:41,229 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.HStore(327): Store=e84bafb3668f028684842276e5650d4c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:41,229 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:41,232 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e84bafb3668f028684842276e5650d4c columnFamilyName cf2 2024-11-20T20:35:41,232 DEBUG [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:41,233 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.HStore(327): Store=e84bafb3668f028684842276e5650d4c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:41,233 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:41,234 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:41,235 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:41,236 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:41,236 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:41,238 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-20T20:35:41,239 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:41,246 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T20:35:41,247 INFO [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e84bafb3668f028684842276e5650d4c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64768558, jitterRate=-0.03487327694892883}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-20T20:35:41,247 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:41,248 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e84bafb3668f028684842276e5650d4c: Running coprocessor pre-open hook at 1732134941223Writing region info on filesystem at 1732134941223Initializing all the Stores at 1732134941225 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134941225Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134941225Cleaning up temporary data from old regions at 1732134941236 (+11 ms)Running coprocessor post-open hooks at 1732134941247 (+11 ms)Region opened successfully at 1732134941248 (+1 ms) 2024-11-20T20:35:41,251 INFO [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., pid=6, masterSystemTime=1732134941212 2024-11-20T20:35:41,255 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:41,255 INFO [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:41,256 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e84bafb3668f028684842276e5650d4c, regionState=OPEN, openSeqNum=2, regionLocation=d7d59561a64e,36639,1732134919021 2024-11-20T20:35:41,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e84bafb3668f028684842276e5650d4c, server=d7d59561a64e,36639,1732134919021 because future has completed 2024-11-20T20:35:41,273 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T20:35:41,273 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e84bafb3668f028684842276e5650d4c, server=d7d59561a64e,36639,1732134919021 in 215 msec 2024-11-20T20:35:41,278 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T20:35:41,278 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e84bafb3668f028684842276e5650d4c, ASSIGN in 403 msec 2024-11-20T20:35:41,280 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T20:35:41,280 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732134941280"}]},"ts":"1732134941280"} 2024-11-20T20:35:41,289 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-20T20:35:41,292 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T20:35:41,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 925 msec 2024-11-20T20:35:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46641 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T20:35:41,523 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-20T20:35:41,524 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-20T20:35:41,526 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T20:35:41,535 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-20T20:35:41,536 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T20:35:41,537 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-20T20:35:41,555 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., hostname=d7d59561a64e,36639,1732134919021, seqNum=2] 2024-11-20T20:35:41,590 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46641 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=e84bafb3668f028684842276e5650d4c, source=d7d59561a64e,36639,1732134919021, destination=d7d59561a64e,33553,1732134919262, warming up region on d7d59561a64e,33553,1732134919262 2024-11-20T20:35:41,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46641 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T20:35:41,592 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46641 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=e84bafb3668f028684842276e5650d4c, source=d7d59561a64e,36639,1732134919021, destination=d7d59561a64e,33553,1732134919262, running balancer 2024-11-20T20:35:41,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46641 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e84bafb3668f028684842276e5650d4c, REOPEN/MOVE 2024-11-20T20:35:41,595 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e84bafb3668f028684842276e5650d4c, REOPEN/MOVE 2024-11-20T20:35:41,598 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e84bafb3668f028684842276e5650d4c, regionState=CLOSING, regionLocation=d7d59561a64e,36639,1732134919021 2024-11-20T20:35:41,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e84bafb3668f028684842276e5650d4c, REOPEN/MOVE because future has completed 2024-11-20T20:35:41,606 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T20:35:41,606 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32779, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T20:35:41,607 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure e84bafb3668f028684842276e5650d4c, server=d7d59561a64e,36639,1732134919021}] 2024-11-20T20:35:41,614 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33553 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33553 {}] regionserver.HRegion(7855): Warmup {ENCODED => e84bafb3668f028684842276e5650d4c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c.', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:35:41,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33553 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:41,620 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:41,624 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e84bafb3668f028684842276e5650d4c columnFamilyName cf1 2024-11-20T20:35:41,624 DEBUG [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:41,625 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.HStore(327): Store=e84bafb3668f028684842276e5650d4c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:41,625 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:41,629 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e84bafb3668f028684842276e5650d4c columnFamilyName cf2 2024-11-20T20:35:41,629 DEBUG [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:41,634 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.HStore(327): Store=e84bafb3668f028684842276e5650d4c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:41,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33553 {}] regionserver.HRegion(1722): Closing e84bafb3668f028684842276e5650d4c, disabling compactions & flushes 2024-11-20T20:35:41,635 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33553 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:41,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33553 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:41,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33553 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. after waiting 0 ms 2024-11-20T20:35:41,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33553 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:41,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46641 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-20T20:35:41,654 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33553 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:41,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33553 {}] regionserver.HRegion(1676): Region close journal for e84bafb3668f028684842276e5650d4c: Waiting for close lock at 1732134941635Disabling compacts and flushes for region at 1732134941635Disabling writes for close at 1732134941635Writing region close event to WAL at 1732134941654 (+19 ms)Closed at 1732134941654 2024-11-20T20:35:41,780 INFO [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:41,781 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T20:35:41,782 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing e84bafb3668f028684842276e5650d4c, disabling compactions & flushes 2024-11-20T20:35:41,782 INFO [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:41,782 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:41,782 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. after waiting 0 ms 2024-11-20T20:35:41,782 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:41,782 INFO [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing e84bafb3668f028684842276e5650d4c 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-20T20:35:41,843 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/.tmp/cf1/cb676e372a4a493aab8683eb6c6bb0a5 is 35, key is r1/cf1:q/1732134941561/Put/seqid=0 2024-11-20T20:35:41,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741887_1065 (size=4783) 2024-11-20T20:35:41,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741887_1065 (size=4783) 2024-11-20T20:35:41,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741887_1065 (size=4783) 2024-11-20T20:35:41,877 INFO [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/.tmp/cf1/cb676e372a4a493aab8683eb6c6bb0a5 2024-11-20T20:35:41,907 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/.tmp/cf1/cb676e372a4a493aab8683eb6c6bb0a5 as hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf1/cb676e372a4a493aab8683eb6c6bb0a5 2024-11-20T20:35:41,937 INFO [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf1/cb676e372a4a493aab8683eb6c6bb0a5, entries=1, sequenceid=5, filesize=4.7 K 2024-11-20T20:35:41,939 INFO [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for e84bafb3668f028684842276e5650d4c in 157ms, sequenceid=5, compaction requested=false 2024-11-20T20:35:41,940 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-20T20:35:41,960 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-20T20:35:41,964 INFO [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:41,964 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for e84bafb3668f028684842276e5650d4c: Waiting for close lock at 1732134941781Running coprocessor pre-close hooks at 1732134941781Disabling compacts and flushes for region at 1732134941781Disabling writes for close at 1732134941782 (+1 ms)Obtaining lock to block concurrent updates at 1732134941782Preparing flush snapshotting stores in e84bafb3668f028684842276e5650d4c at 1732134941782Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732134941782Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. at 1732134941788 (+6 ms)Flushing e84bafb3668f028684842276e5650d4c/cf1: creating writer at 1732134941789 (+1 ms)Flushing e84bafb3668f028684842276e5650d4c/cf1: appending metadata at 1732134941843 (+54 ms)Flushing e84bafb3668f028684842276e5650d4c/cf1: closing flushed file at 1732134941843Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50f068c0: reopening flushed file at 1732134941906 (+63 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for e84bafb3668f028684842276e5650d4c in 157ms, sequenceid=5, compaction requested=false at 1732134941939 (+33 ms)Writing region close event to WAL at 1732134941946 (+7 ms)Running coprocessor post-close hooks at 1732134941961 (+15 ms)Closed at 1732134941964 (+3 ms) 2024-11-20T20:35:41,964 INFO [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding e84bafb3668f028684842276e5650d4c move to d7d59561a64e,33553,1732134919262 record at close sequenceid=5 2024-11-20T20:35:41,968 INFO [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:41,969 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e84bafb3668f028684842276e5650d4c, regionState=CLOSED 2024-11-20T20:35:41,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure e84bafb3668f028684842276e5650d4c, server=d7d59561a64e,36639,1732134919021 because future has completed 2024-11-20T20:35:41,982 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T20:35:41,982 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure e84bafb3668f028684842276e5650d4c, server=d7d59561a64e,36639,1732134919021 in 368 msec 2024-11-20T20:35:41,984 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e84bafb3668f028684842276e5650d4c, REOPEN/MOVE; state=CLOSED, location=d7d59561a64e,33553,1732134919262; forceNewPlan=false, retain=false 2024-11-20T20:35:42,136 INFO [d7d59561a64e:46641 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T20:35:42,140 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e84bafb3668f028684842276e5650d4c, regionState=OPENING, regionLocation=d7d59561a64e,33553,1732134919262 2024-11-20T20:35:42,156 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e84bafb3668f028684842276e5650d4c, REOPEN/MOVE because future has completed 2024-11-20T20:35:42,160 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure e84bafb3668f028684842276e5650d4c, server=d7d59561a64e,33553,1732134919262}] 2024-11-20T20:35:42,344 INFO [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:42,345 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => e84bafb3668f028684842276e5650d4c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c.', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:35:42,345 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:42,345 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:42,345 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:42,345 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:42,368 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:42,377 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e84bafb3668f028684842276e5650d4c columnFamilyName cf1 2024-11-20T20:35:42,377 DEBUG [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:42,426 DEBUG [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf1/cb676e372a4a493aab8683eb6c6bb0a5 2024-11-20T20:35:42,426 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.HStore(327): Store=e84bafb3668f028684842276e5650d4c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:42,426 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:42,428 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e84bafb3668f028684842276e5650d4c columnFamilyName cf2 2024-11-20T20:35:42,428 DEBUG [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:42,430 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.HStore(327): Store=e84bafb3668f028684842276e5650d4c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:42,430 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:42,432 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:42,434 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:42,435 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:42,435 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:42,441 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-20T20:35:42,446 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:42,448 INFO [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened e84bafb3668f028684842276e5650d4c; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70394790, jitterRate=0.048964112997055054}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-20T20:35:42,448 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:42,450 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for e84bafb3668f028684842276e5650d4c: Running coprocessor pre-open hook at 1732134942346Writing region info on filesystem at 1732134942346Initializing all the Stores at 1732134942361 (+15 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134942361Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134942365 (+4 ms)Cleaning up temporary data from old regions at 1732134942435 (+70 ms)Running coprocessor post-open hooks at 1732134942448 (+13 ms)Region opened successfully at 1732134942450 (+2 ms) 2024-11-20T20:35:42,452 INFO [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., pid=9, masterSystemTime=1732134942327 2024-11-20T20:35:42,472 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:42,472 INFO [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:42,477 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e84bafb3668f028684842276e5650d4c, regionState=OPEN, openSeqNum=9, regionLocation=d7d59561a64e,33553,1732134919262 2024-11-20T20:35:42,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure e84bafb3668f028684842276e5650d4c, server=d7d59561a64e,33553,1732134919262 because future has completed 2024-11-20T20:35:42,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-20T20:35:42,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure e84bafb3668f028684842276e5650d4c, server=d7d59561a64e,33553,1732134919262 in 345 msec 2024-11-20T20:35:42,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e84bafb3668f028684842276e5650d4c, REOPEN/MOVE in 925 msec 2024-11-20T20:35:42,560 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T20:35:42,563 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T20:35:42,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36639 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:43170 deadline: 1732135002567, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=d7d59561a64e port=33553 startCode=1732134919262. As of locationSeqNum=5. 2024-11-20T20:35:42,597 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., hostname=d7d59561a64e,36639,1732134919021, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., hostname=d7d59561a64e,36639,1732134919021, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=d7d59561a64e port=33553 startCode=1732134919262. As of locationSeqNum=5. 2024-11-20T20:35:42,598 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., hostname=d7d59561a64e,36639,1732134919021, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=d7d59561a64e port=33553 startCode=1732134919262. As of locationSeqNum=5. 2024-11-20T20:35:42,598 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., hostname=d7d59561a64e,36639,1732134919021, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., hostname=d7d59561a64e,33553,1732134919262, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=d7d59561a64e port=33553 startCode=1732134919262. As of locationSeqNum=5. 2024-11-20T20:35:42,711 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T20:35:42,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T20:35:42,726 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e84bafb3668f028684842276e5650d4c 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-20T20:35:42,750 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/.tmp/cf1/806e202afbbf40238fe32833e612a38e is 29, key is r1/cf1:/1732134942715/DeleteFamily/seqid=0 2024-11-20T20:35:42,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741888_1066 (size=4906) 2024-11-20T20:35:42,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741888_1066 (size=4906) 2024-11-20T20:35:42,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741888_1066 (size=4906) 2024-11-20T20:35:42,776 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/.tmp/cf1/806e202afbbf40238fe32833e612a38e 2024-11-20T20:35:42,782 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 806e202afbbf40238fe32833e612a38e 2024-11-20T20:35:42,798 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/.tmp/cf2/9430c12afec5466e8c97a9b76aa78c40 is 29, key is r1/cf2:/1732134942715/DeleteFamily/seqid=0 2024-11-20T20:35:42,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741889_1067 (size=4906) 2024-11-20T20:35:42,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741889_1067 (size=4906) 2024-11-20T20:35:42,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741889_1067 (size=4906) 2024-11-20T20:35:42,807 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/.tmp/cf2/9430c12afec5466e8c97a9b76aa78c40 2024-11-20T20:35:42,815 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9430c12afec5466e8c97a9b76aa78c40 2024-11-20T20:35:42,816 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/.tmp/cf1/806e202afbbf40238fe32833e612a38e as hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf1/806e202afbbf40238fe32833e612a38e 2024-11-20T20:35:42,823 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 806e202afbbf40238fe32833e612a38e 2024-11-20T20:35:42,823 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf1/806e202afbbf40238fe32833e612a38e, entries=1, sequenceid=12, filesize=4.8 K 2024-11-20T20:35:42,825 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/.tmp/cf2/9430c12afec5466e8c97a9b76aa78c40 as hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf2/9430c12afec5466e8c97a9b76aa78c40 2024-11-20T20:35:42,831 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9430c12afec5466e8c97a9b76aa78c40 2024-11-20T20:35:42,831 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf2/9430c12afec5466e8c97a9b76aa78c40, entries=1, sequenceid=12, filesize=4.8 K 2024-11-20T20:35:42,833 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for e84bafb3668f028684842276e5650d4c in 107ms, sequenceid=12, compaction requested=false 2024-11-20T20:35:42,833 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e84bafb3668f028684842276e5650d4c: 2024-11-20T20:35:42,836 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T20:35:42,837 DEBUG [Time-limited test {}] regionserver.HStore(1541): e84bafb3668f028684842276e5650d4c/cf1 is initiating major compaction (all files) 2024-11-20T20:35:42,837 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T20:35:42,837 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:42,837 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of e84bafb3668f028684842276e5650d4c/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:42,838 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf1/cb676e372a4a493aab8683eb6c6bb0a5, hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf1/806e202afbbf40238fe32833e612a38e] into tmpdir=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/.tmp, totalSize=9.5 K 2024-11-20T20:35:42,839 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting cb676e372a4a493aab8683eb6c6bb0a5, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732134941561 2024-11-20T20:35:42,840 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 806e202afbbf40238fe32833e612a38e, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-20T20:35:42,853 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): e84bafb3668f028684842276e5650d4c#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T20:35:42,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741890_1068 (size=4626) 2024-11-20T20:35:42,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741890_1068 (size=4626) 2024-11-20T20:35:42,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741890_1068 (size=4626) 2024-11-20T20:35:42,874 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/.tmp/cf1/abd38df55e1c456c90a12675eeae07e2 as hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf1/abd38df55e1c456c90a12675eeae07e2 2024-11-20T20:35:42,893 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in e84bafb3668f028684842276e5650d4c/cf1 of e84bafb3668f028684842276e5650d4c into abd38df55e1c456c90a12675eeae07e2(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T20:35:42,893 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for e84bafb3668f028684842276e5650d4c: 2024-11-20T20:35:42,893 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-20T20:35:42,893 DEBUG [Time-limited test {}] regionserver.HStore(1541): e84bafb3668f028684842276e5650d4c/cf2 is initiating major compaction (all files) 2024-11-20T20:35:42,893 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T20:35:42,893 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T20:35:42,894 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of e84bafb3668f028684842276e5650d4c/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:42,894 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf2/9430c12afec5466e8c97a9b76aa78c40] into tmpdir=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/.tmp, totalSize=4.8 K 2024-11-20T20:35:42,895 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 9430c12afec5466e8c97a9b76aa78c40, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-20T20:35:42,905 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): e84bafb3668f028684842276e5650d4c#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T20:35:42,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741891_1069 (size=4592) 2024-11-20T20:35:42,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741891_1069 (size=4592) 2024-11-20T20:35:42,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741891_1069 (size=4592) 2024-11-20T20:35:42,945 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/.tmp/cf2/c63971d5759c499faadf4cf40b2962a5 as hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf2/c63971d5759c499faadf4cf40b2962a5 2024-11-20T20:35:42,955 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in e84bafb3668f028684842276e5650d4c/cf2 of e84bafb3668f028684842276e5650d4c into c63971d5759c499faadf4cf40b2962a5(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T20:35:42,955 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for e84bafb3668f028684842276e5650d4c: 2024-11-20T20:35:42,962 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46641 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=e84bafb3668f028684842276e5650d4c, source=d7d59561a64e,33553,1732134919262, destination=d7d59561a64e,36639,1732134919021, warming up region on d7d59561a64e,36639,1732134919021 2024-11-20T20:35:42,962 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46641 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=e84bafb3668f028684842276e5650d4c, source=d7d59561a64e,33553,1732134919262, destination=d7d59561a64e,36639,1732134919021, running balancer 2024-11-20T20:35:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46641 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e84bafb3668f028684842276e5650d4c, REOPEN/MOVE 2024-11-20T20:35:42,965 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e84bafb3668f028684842276e5650d4c, REOPEN/MOVE 2024-11-20T20:35:42,967 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e84bafb3668f028684842276e5650d4c, regionState=CLOSING, regionLocation=d7d59561a64e,33553,1732134919262 2024-11-20T20:35:42,967 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36639 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36639 {}] regionserver.HRegion(7855): Warmup {ENCODED => e84bafb3668f028684842276e5650d4c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c.', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:35:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36639 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:42,968 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:42,969 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e84bafb3668f028684842276e5650d4c columnFamilyName cf1 2024-11-20T20:35:42,969 DEBUG [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:42,971 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e84bafb3668f028684842276e5650d4c, REOPEN/MOVE because future has completed 2024-11-20T20:35:42,971 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T20:35:42,971 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure e84bafb3668f028684842276e5650d4c, server=d7d59561a64e,33553,1732134919262}] 2024-11-20T20:35:42,981 INFO [StoreFileOpener-e84bafb3668f028684842276e5650d4c-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 806e202afbbf40238fe32833e612a38e 2024-11-20T20:35:42,981 DEBUG [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf1/806e202afbbf40238fe32833e612a38e 2024-11-20T20:35:42,988 DEBUG [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf1/abd38df55e1c456c90a12675eeae07e2 2024-11-20T20:35:42,995 DEBUG [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf1/cb676e372a4a493aab8683eb6c6bb0a5 2024-11-20T20:35:42,995 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.HStore(327): Store=e84bafb3668f028684842276e5650d4c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:42,996 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:42,997 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e84bafb3668f028684842276e5650d4c columnFamilyName cf2 2024-11-20T20:35:42,997 DEBUG [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:43,005 INFO [StoreFileOpener-e84bafb3668f028684842276e5650d4c-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9430c12afec5466e8c97a9b76aa78c40 2024-11-20T20:35:43,005 DEBUG [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf2/9430c12afec5466e8c97a9b76aa78c40 2024-11-20T20:35:43,010 DEBUG [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf2/c63971d5759c499faadf4cf40b2962a5 2024-11-20T20:35:43,011 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.HStore(327): Store=e84bafb3668f028684842276e5650d4c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36639 {}] regionserver.HRegion(1722): Closing e84bafb3668f028684842276e5650d4c, disabling compactions & flushes 2024-11-20T20:35:43,011 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36639 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36639 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36639 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. after waiting 0 ms 2024-11-20T20:35:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36639 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:43,012 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36639 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36639 {}] regionserver.HRegion(1676): Region close journal for e84bafb3668f028684842276e5650d4c: Waiting for close lock at 1732134943011Disabling compacts and flushes for region at 1732134943011Disabling writes for close at 1732134943011Writing region close event to WAL at 1732134943012 (+1 ms)Closed at 1732134943012 2024-11-20T20:35:43,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46641 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-20T20:35:43,132 INFO [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:43,132 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T20:35:43,132 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing e84bafb3668f028684842276e5650d4c, disabling compactions & flushes 2024-11-20T20:35:43,132 INFO [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:43,132 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:43,132 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. after waiting 0 ms 2024-11-20T20:35:43,132 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:43,136 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf1/cb676e372a4a493aab8683eb6c6bb0a5, hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf1/806e202afbbf40238fe32833e612a38e] to archive 2024-11-20T20:35:43,140 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T20:35:43,146 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf1/cb676e372a4a493aab8683eb6c6bb0a5 to hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf1/cb676e372a4a493aab8683eb6c6bb0a5 2024-11-20T20:35:43,150 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf1/806e202afbbf40238fe32833e612a38e to hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf1/806e202afbbf40238fe32833e612a38e 2024-11-20T20:35:43,171 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf2/9430c12afec5466e8c97a9b76aa78c40] to archive 2024-11-20T20:35:43,172 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T20:35:43,175 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf2/9430c12afec5466e8c97a9b76aa78c40 to hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf2/9430c12afec5466e8c97a9b76aa78c40 2024-11-20T20:35:43,189 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-20T20:35:43,190 INFO [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:43,190 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for e84bafb3668f028684842276e5650d4c: Waiting for close lock at 1732134943132Running coprocessor pre-close hooks at 1732134943132Disabling compacts and flushes for region at 1732134943132Disabling writes for close at 1732134943132Writing region close event to WAL at 1732134943177 (+45 ms)Running coprocessor post-close hooks at 1732134943190 (+13 ms)Closed at 1732134943190 2024-11-20T20:35:43,190 INFO [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding e84bafb3668f028684842276e5650d4c move to d7d59561a64e,36639,1732134919021 record at close sequenceid=12 2024-11-20T20:35:43,199 INFO [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:43,203 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e84bafb3668f028684842276e5650d4c, regionState=CLOSED 2024-11-20T20:35:43,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure e84bafb3668f028684842276e5650d4c, server=d7d59561a64e,33553,1732134919262 because future has completed 2024-11-20T20:35:43,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-20T20:35:43,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure e84bafb3668f028684842276e5650d4c, server=d7d59561a64e,33553,1732134919262 in 241 msec 2024-11-20T20:35:43,225 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e84bafb3668f028684842276e5650d4c, REOPEN/MOVE; state=CLOSED, location=d7d59561a64e,36639,1732134919021; forceNewPlan=false, retain=false 2024-11-20T20:35:43,376 INFO [d7d59561a64e:46641 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T20:35:43,376 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e84bafb3668f028684842276e5650d4c, regionState=OPENING, regionLocation=d7d59561a64e,36639,1732134919021 2024-11-20T20:35:43,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e84bafb3668f028684842276e5650d4c, REOPEN/MOVE because future has completed 2024-11-20T20:35:43,380 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure e84bafb3668f028684842276e5650d4c, server=d7d59561a64e,36639,1732134919021}] 2024-11-20T20:35:43,547 INFO [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:43,547 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => e84bafb3668f028684842276e5650d4c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c.', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:35:43,548 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:43,548 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:43,548 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:43,548 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:43,549 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:43,550 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e84bafb3668f028684842276e5650d4c columnFamilyName cf1 2024-11-20T20:35:43,550 DEBUG [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:43,556 DEBUG [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf1/abd38df55e1c456c90a12675eeae07e2 2024-11-20T20:35:43,556 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.HStore(327): Store=e84bafb3668f028684842276e5650d4c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:43,556 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:43,557 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e84bafb3668f028684842276e5650d4c columnFamilyName cf2 2024-11-20T20:35:43,558 DEBUG [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:43,564 DEBUG [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf2/c63971d5759c499faadf4cf40b2962a5 2024-11-20T20:35:43,564 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.HStore(327): Store=e84bafb3668f028684842276e5650d4c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:43,564 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:43,565 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:43,566 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:43,567 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:43,567 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:43,568 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-20T20:35:43,571 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:43,572 INFO [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened e84bafb3668f028684842276e5650d4c; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59010168, jitterRate=-0.12067997455596924}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-20T20:35:43,572 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:43,573 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for e84bafb3668f028684842276e5650d4c: Running coprocessor pre-open hook at 1732134943548Writing region info on filesystem at 1732134943548Initializing all the Stores at 1732134943549 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134943549Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134943549Cleaning up temporary data from old regions at 1732134943567 (+18 ms)Running coprocessor post-open hooks at 1732134943572 (+5 ms)Region opened successfully at 1732134943573 (+1 ms) 2024-11-20T20:35:43,574 INFO [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., pid=12, masterSystemTime=1732134943542 2024-11-20T20:35:43,577 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:43,577 INFO [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:43,578 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e84bafb3668f028684842276e5650d4c, regionState=OPEN, openSeqNum=18, regionLocation=d7d59561a64e,36639,1732134919021 2024-11-20T20:35:43,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure e84bafb3668f028684842276e5650d4c, server=d7d59561a64e,36639,1732134919021 because future has completed 2024-11-20T20:35:43,586 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-20T20:35:43,586 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure e84bafb3668f028684842276e5650d4c, server=d7d59561a64e,36639,1732134919021 in 203 msec 2024-11-20T20:35:43,589 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e84bafb3668f028684842276e5650d4c, REOPEN/MOVE in 624 msec 2024-11-20T20:35:43,617 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T20:35:43,619 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T20:35:43,621 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server d7d59561a64e,36639,1732134919021: testing ***** 2024-11-20T20:35:43,621 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-20T20:35:43,623 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-20T20:35:43,624 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-20T20:35:43,631 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-20T20:35:43,633 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-20T20:35:43,646 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 340378168 }, "NonHeapMemoryUsage": { "committed": 171704320, "init": 7667712, "max": -1, "used": 169083152 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "d7d59561a64e", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 4, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 10271, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 49, "ProcessCallTime_mean": 7, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 3, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 13, "ProcessCallTime_95th_percentile": 47, "ProcessCallTime_98th_percentile": 49, "ProcessCallTime_99th_percentile": 49, "ProcessCallTime_99.9th_percentile": 49, "ProcessCallTime_TimeRangeCount_0-1": 6, "ProcessCallTime_TimeRangeCount_1-3": 10, "ProcessCallTime_TimeRangeCount_3-10": 11, "ProcessCallTime_TimeRangeCount_10-30": 2, "ProcessCallTime_TimeRangeCount_30-100": 2, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 3, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 1, "QueueCallTime_90th_percentile": 1, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 3, "QueueCallTime_99th_percentile": 3, "QueueCallTime_99.9th_percentile": 3, "QueueCallTime_TimeRangeCount_0-1": 30, "QueueCallTime_TimeRangeCount_1-3": 1, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 1, "TotalCallTime_max": 49, "TotalCallTime_mean": 7, "TotalCallTime_25th_percentile": 2, "TotalCallTime_median": 3, "TotalCallTime_75th_percentile": 9, "TotalCallTime_90th_percentile": 14, "TotalCallTime_95th_percentile": 47, "TotalCallTime_98th_percentile": 49, "TotalCallTime_99th_percentile": 49, "TotalCallTime_99.9th_percentile": 49, "TotalCallTime_TimeRangeCount_0-1": 5, "TotalCallTime_TimeRangeCount_1-3": 11, "TotalCallTime_TimeRangeCount_3-10": 11, "TotalCallTime_TimeRangeCount_10-30": 2, "TotalCallTime_TimeRangeCount_30-100": 2, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1138, "ResponseSize_mean": 159, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 795, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1069, "ResponseSize_99.9th_percentile": 1131, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5772 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "d7d59561a64e", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:64093", "tag.serverName": "d7d59561a64e,36243,1732134919381", "tag.clusterId": "92b6d327-3593-434a-bf0b-fee82840fcda", "tag.Context": "regionserver", "tag.Hostname": "d7d59561a64e", "regionCount": 0, "storeCount": 0, "hlogFileCount": 4, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732134919381, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227243, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 1.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-20T20:35:43,650 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46641 {}] master.MasterRpcServices(700): d7d59561a64e,36639,1732134919021 reported a fatal error: ***** ABORTING region server d7d59561a64e,36639,1732134919021: testing ***** 2024-11-20T20:35:43,653 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd7d59561a64e,36639,1732134919021' ***** 2024-11-20T20:35:43,653 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-20T20:35:43,653 INFO [RS:0;d7d59561a64e:36639 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T20:35:43,653 INFO [RS:0;d7d59561a64e:36639 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-20T20:35:43,653 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T20:35:43,654 INFO [RS:0;d7d59561a64e:36639 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-20T20:35:43,654 INFO [RS:0;d7d59561a64e:36639 {}] regionserver.HRegionServer(3091): Received CLOSE for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:43,654 INFO [RS:0;d7d59561a64e:36639 {}] regionserver.HRegionServer(956): aborting server d7d59561a64e,36639,1732134919021 2024-11-20T20:35:43,654 INFO [RS:0;d7d59561a64e:36639 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T20:35:43,655 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e84bafb3668f028684842276e5650d4c, disabling compactions & flushes 2024-11-20T20:35:43,655 INFO [RS:0;d7d59561a64e:36639 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d7d59561a64e:36639. 2024-11-20T20:35:43,655 INFO [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:43,655 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:43,655 DEBUG [RS:0;d7d59561a64e:36639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T20:35:43,655 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. after waiting 0 ms 2024-11-20T20:35:43,655 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:43,655 DEBUG [RS:0;d7d59561a64e:36639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T20:35:43,656 INFO [RS:0;d7d59561a64e:36639 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T20:35:43,656 INFO [RS:0;d7d59561a64e:36639 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T20:35:43,656 INFO [RS:0;d7d59561a64e:36639 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T20:35:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33553 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:44572 deadline: 1732135003656, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=d7d59561a64e port=36639 startCode=1732134919021. As of locationSeqNum=12. 2024-11-20T20:35:43,661 INFO [RS:0;d7d59561a64e:36639 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T20:35:43,661 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., hostname=d7d59561a64e,33553,1732134919262, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., hostname=d7d59561a64e,33553,1732134919262, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=d7d59561a64e port=36639 startCode=1732134919021. As of locationSeqNum=12. 2024-11-20T20:35:43,661 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., hostname=d7d59561a64e,33553,1732134919262, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=d7d59561a64e port=36639 startCode=1732134919021. As of locationSeqNum=12. 2024-11-20T20:35:43,661 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., hostname=d7d59561a64e,33553,1732134919262, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., hostname=d7d59561a64e,36639,1732134919021, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=d7d59561a64e port=36639 startCode=1732134919021. As of locationSeqNum=12. 2024-11-20T20:35:43,663 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server d7d59561a64e,36639,1732134919021 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:35:43,665 INFO [RS:0;d7d59561a64e:36639 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-20T20:35:43,665 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,36639,1732134919021, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,36639,1732134919021, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server d7d59561a64e,36639,1732134919021 aborting 2024-11-20T20:35:43,665 DEBUG [RS:0;d7d59561a64e:36639 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, e84bafb3668f028684842276e5650d4c=testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c.} 2024-11-20T20:35:43,665 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,36639,1732134919021, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server d7d59561a64e,36639,1732134919021 aborting 2024-11-20T20:35:43,665 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,36639,1732134919021, seqNum=-1 from cache 2024-11-20T20:35:43,665 DEBUG [RS:0;d7d59561a64e:36639 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:43,668 DEBUG [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T20:35:43,668 INFO [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T20:35:43,669 DEBUG [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T20:35:43,669 DEBUG [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T20:35:43,669 DEBUG [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T20:35:43,669 INFO [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:43,669 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e84bafb3668f028684842276e5650d4c: Waiting for close lock at 1732134943654Running coprocessor pre-close hooks at 1732134943654Disabling compacts and flushes for region at 1732134943654Disabling writes for close at 1732134943655 (+1 ms)Writing region close event to WAL at 1732134943669 (+14 ms)Running coprocessor post-close hooks at 1732134943669Closed at 1732134943669 2024-11-20T20:35:43,669 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:43,672 ERROR [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-20T20:35:43,672 DEBUG [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T20:35:43,672 INFO [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T20:35:43,673 DEBUG [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732134943668Running coprocessor pre-close hooks at 1732134943668Disabling compacts and flushes for region at 1732134943668Disabling writes for close at 1732134943669 (+1 ms)Writing region close event to WAL at 1732134943672 (+3 ms)Running coprocessor post-close hooks at 1732134943672Closed at 1732134943672 2024-11-20T20:35:43,673 DEBUG [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T20:35:43,724 INFO [regionserver/d7d59561a64e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T20:35:43,770 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T20:35:43,772 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,36639,1732134919021, seqNum=-1] 2024-11-20T20:35:43,773 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server d7d59561a64e,36639,1732134919021 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:35:43,774 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,36639,1732134919021, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,36639,1732134919021, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server d7d59561a64e,36639,1732134919021 aborting 2024-11-20T20:35:43,774 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,36639,1732134919021, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server d7d59561a64e,36639,1732134919021 aborting 2024-11-20T20:35:43,774 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,36639,1732134919021, seqNum=-1 from cache 2024-11-20T20:35:43,866 INFO [RS:0;d7d59561a64e:36639 {}] regionserver.HRegionServer(976): stopping server d7d59561a64e,36639,1732134919021; all regions closed. 2024-11-20T20:35:43,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741836_1012 (size=2747) 2024-11-20T20:35:43,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741836_1012 (size=2747) 2024-11-20T20:35:43,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741836_1012 (size=2747) 2024-11-20T20:35:43,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741835_1011 (size=1677) 2024-11-20T20:35:43,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741835_1011 (size=1677) 2024-11-20T20:35:43,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741835_1011 (size=1677) 2024-11-20T20:35:43,885 DEBUG [RS:0;d7d59561a64e:36639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T20:35:43,885 INFO [RS:0;d7d59561a64e:36639 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T20:35:43,888 INFO [RS:0;d7d59561a64e:36639 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T20:35:43,889 INFO [RS:0;d7d59561a64e:36639 {}] hbase.ChoreService(370): Chore service for: regionserver/d7d59561a64e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-20T20:35:43,889 INFO [RS:0;d7d59561a64e:36639 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T20:35:43,889 INFO [regionserver/d7d59561a64e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T20:35:43,889 INFO [RS:0;d7d59561a64e:36639 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36639 2024-11-20T20:35:43,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36639-0x10159f54fc60001, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d7d59561a64e,36639,1732134919021 2024-11-20T20:35:43,924 INFO [RS:0;d7d59561a64e:36639 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T20:35:43,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T20:35:43,935 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d7d59561a64e,36639,1732134919021] 2024-11-20T20:35:43,945 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d7d59561a64e,36639,1732134919021 already deleted, retry=false 2024-11-20T20:35:43,945 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of d7d59561a64e,36639,1732134919021 on d7d59561a64e,46641,1732134918030 2024-11-20T20:35:43,949 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure d7d59561a64e,36639,1732134919021, splitWal=true, meta=true 2024-11-20T20:35:43,952 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=13 for d7d59561a64e,36639,1732134919021 (carryingMeta=true) d7d59561a64e,36639,1732134919021/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@54efff0c[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-20T20:35:43,953 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure d7d59561a64e,36639,1732134919021, splitWal=true, meta=true 2024-11-20T20:35:43,955 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure d7d59561a64e,36639,1732134919021, splitWal=true, meta=true, isMeta: true 2024-11-20T20:35:43,957 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting 2024-11-20T20:35:43,958 INFO [PEWorker-2 {}] master.SplitWALManager(105): d7d59561a64e,36639,1732134919021 WAL count=1, meta=true 2024-11-20T20:35:43,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta}] 2024-11-20T20:35:43,967 DEBUG [PEWorker-5 {}] master.SplitWALManager(158): Acquired split WAL worker=d7d59561a64e,36243,1732134919381 2024-11-20T20:35:43,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta, worker=d7d59561a64e,36243,1732134919381}] 2024-11-20T20:35:43,981 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T20:35:43,984 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,36639,1732134919021, seqNum=-1] 2024-11-20T20:35:43,985 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server d7d59561a64e:36639 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: d7d59561a64e/172.17.0.2:36639 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:35:43,986 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,36639,1732134919021, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,36639,1732134919021, seqNum=-1, error=java.net.ConnectException: Call to address=d7d59561a64e:36639 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: d7d59561a64e/172.17.0.2:36639 2024-11-20T20:35:43,986 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,36639,1732134919021, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-20T20:35:43,986 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,36639,1732134919021, seqNum=-1 from cache 2024-11-20T20:35:43,987 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address d7d59561a64e:36639 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: d7d59561a64e/172.17.0.2:36639 2024-11-20T20:35:44,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36639-0x10159f54fc60001, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T20:35:44,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36639-0x10159f54fc60001, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T20:35:44,035 INFO [RS:0;d7d59561a64e:36639 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T20:35:44,036 INFO [RS:0;d7d59561a64e:36639 {}] regionserver.HRegionServer(1031): Exiting; stopping=d7d59561a64e,36639,1732134919021; zookeeper connection closed. 2024-11-20T20:35:44,036 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@570e425b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@570e425b 2024-11-20T20:35:44,136 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T20:35:44,138 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60581, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T20:35:44,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-20T20:35:44,171 INFO [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta, size=2.7 K (2747bytes) 2024-11-20T20:35:44,171 INFO [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta 2024-11-20T20:35:44,172 INFO [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta after 0ms 2024-11-20T20:35:44,174 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:44,174 INFO [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta took 3ms 2024-11-20T20:35:44,183 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-20T20:35:44,184 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta so closing down 2024-11-20T20:35:44,184 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T20:35:44,188 INFO [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta.temp 2024-11-20T20:35:44,290 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T20:35:44,291 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,36639,1732134919021, seqNum=-1] 2024-11-20T20:35:44,292 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to d7d59561a64e:36639 this server is in the failed servers list 2024-11-20T20:35:44,293 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,36639,1732134919021, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,36639,1732134919021, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=d7d59561a64e:36639 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: d7d59561a64e:36639 2024-11-20T20:35:44,293 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,36639,1732134919021, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: d7d59561a64e:36639 2024-11-20T20:35:44,293 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,36639,1732134919021, seqNum=-1 from cache 2024-11-20T20:35:44,329 INFO [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta.temp 2024-11-20T20:35:44,330 INFO [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T20:35:44,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741892_1070 (size=2527) 2024-11-20T20:35:44,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741892_1070 (size=2527) 2024-11-20T20:35:44,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741892_1070 (size=2527) 2024-11-20T20:35:44,373 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-20T20:35:44,380 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta.temp to hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-20T20:35:44,381 INFO [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 205 ms; skipped=1; WAL=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta, size=2.7 K, length=2747, corrupted=false, cancelled=false 2024-11-20T20:35:44,381 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta, journal: Splitting hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta, size=2.7 K (2747bytes) at 1732134944171Finishing writing output for hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta so closing down at 1732134944184 (+13 ms)Creating recovered edits writer path=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta.temp at 1732134944329 (+145 ms)3 split writer threads finished at 1732134944330 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732134944373 (+43 ms)Rename recovered edits hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta.temp to hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1732134944380 (+7 ms)Processed 16 edits across 1 Regions in 205 ms; skipped=1; WAL=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta, size=2.7 K, length=2747, corrupted=false, cancelled=false at 1732134944381 (+1 ms) 2024-11-20T20:35:44,381 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta 2024-11-20T20:35:44,383 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-20T20:35:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46641 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-20T20:35:44,398 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta to hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/oldWALs 2024-11-20T20:35:44,403 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-20T20:35:44,403 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta, worker=d7d59561a64e,36243,1732134919381 in 427 msec 2024-11-20T20:35:44,405 DEBUG [PEWorker-4 {}] master.SplitWALManager(172): Release split WAL worker=d7d59561a64e,36243,1732134919381 2024-11-20T20:35:44,409 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-20T20:35:44,409 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure d7d59561a64e%2C36639%2C1732134919021.meta.1732134923005.meta, worker=d7d59561a64e,36243,1732134919381 in 446 msec 2024-11-20T20:35:44,413 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting dir is empty, no logs to split. 2024-11-20T20:35:44,413 INFO [PEWorker-2 {}] master.SplitWALManager(105): d7d59561a64e,36639,1732134919021 WAL count=0, meta=true 2024-11-20T20:35:44,413 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if d7d59561a64e,36639,1732134919021 WAL splitting is done? wals=0, meta=true 2024-11-20T20:35:44,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T20:35:44,418 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T20:35:44,420 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T20:35:44,570 DEBUG [d7d59561a64e:46641 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-20T20:35:44,571 DEBUG [d7d59561a64e:46641 {}] balancer.BalancerClusterState(204): Hosts are {d7d59561a64e=0} racks are {/default-rack=0} 2024-11-20T20:35:44,571 DEBUG [d7d59561a64e:46641 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T20:35:44,571 DEBUG [d7d59561a64e:46641 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T20:35:44,571 DEBUG [d7d59561a64e:46641 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T20:35:44,571 DEBUG [d7d59561a64e:46641 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T20:35:44,571 INFO [d7d59561a64e:46641 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T20:35:44,571 INFO [d7d59561a64e:46641 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T20:35:44,571 DEBUG [d7d59561a64e:46641 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T20:35:44,571 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d7d59561a64e,33553,1732134919262 2024-11-20T20:35:44,573 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d7d59561a64e,33553,1732134919262, state=OPENING 2024-11-20T20:35:44,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36243-0x10159f54fc60003, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T20:35:44,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33553-0x10159f54fc60002, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T20:35:44,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T20:35:44,587 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T20:35:44,587 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T20:35:44,587 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T20:35:44,587 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T20:35:44,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d7d59561a64e,33553,1732134919262}] 2024-11-20T20:35:44,755 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T20:35:44,755 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:44,755 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T20:35:44,758 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d7d59561a64e%2C33553%2C1732134919262.meta, suffix=.meta, logDir=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,33553,1732134919262, archiveDir=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/oldWALs, maxLogs=32 2024-11-20T20:35:44,775 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,33553,1732134919262/d7d59561a64e%2C33553%2C1732134919262.meta.1732134944758.meta, exclude list is [], retry=0 2024-11-20T20:35:44,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:44,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:44,780 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:44,792 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,33553,1732134919262/d7d59561a64e%2C33553%2C1732134919262.meta.1732134944758.meta 2024-11-20T20:35:44,792 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:39759:39759),(127.0.0.1/127.0.0.1:38233:38233)] 2024-11-20T20:35:44,793 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:35:44,793 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T20:35:44,793 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T20:35:44,794 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T20:35:44,794 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T20:35:44,794 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:44,794 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T20:35:44,794 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T20:35:44,797 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T20:35:44,800 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T20:35:44,800 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:44,801 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T20:35:44,801 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T20:35:44,802 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T20:35:44,802 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:44,803 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T20:35:44,803 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T20:35:44,804 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T20:35:44,805 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T20:35:44,805 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:44,806 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T20:35:44,806 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T20:35:44,807 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T20:35:44,807 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:44,808 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T20:35:44,808 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T20:35:44,809 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740 2024-11-20T20:35:44,811 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740 2024-11-20T20:35:44,811 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,33553,1732134919262, seqNum=-1] 2024-11-20T20:35:44,812 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-20T20:35:44,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33553 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Get size: 119 connection: 172.17.0.2:44572 deadline: 1732135004812, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on d7d59561a64e,33553,1732134919262 2024-11-20T20:35:44,817 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/recovered.edits/0000000000000000018: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:44,817 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,33553,1732134919262, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,33553,1732134919262, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on d7d59561a64e,33553,1732134919262 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T20:35:44,817 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,33553,1732134919262, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on d7d59561a64e,33553,1732134919262 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T20:35:44,817 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,33553,1732134919262, seqNum=-1 from cache 2024-11-20T20:35:44,820 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-20T20:35:44,820 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-20T20:35:44,854 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/.tmp/info/58051d9b853d4f248fdf570a1aeb694e is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c./info:regioninfo/1732134943577/Put/seqid=0 2024-11-20T20:35:44,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741894_1072 (size=11177) 2024-11-20T20:35:44,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741894_1072 (size=11177) 2024-11-20T20:35:44,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741894_1072 (size=11177) 2024-11-20T20:35:45,279 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/.tmp/info/58051d9b853d4f248fdf570a1aeb694e 2024-11-20T20:35:45,304 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/.tmp/ns/6d062c730982413792aa3dae312950e1 is 43, key is default/ns:d/1732134923262/Put/seqid=0 2024-11-20T20:35:45,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741895_1073 (size=5153) 2024-11-20T20:35:45,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741895_1073 (size=5153) 2024-11-20T20:35:45,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741895_1073 (size=5153) 2024-11-20T20:35:45,311 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/.tmp/ns/6d062c730982413792aa3dae312950e1 2024-11-20T20:35:45,337 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/.tmp/table/6b58909dd9d54b91bfecdafd1815baea is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732134941280/Put/seqid=0 2024-11-20T20:35:45,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741896_1074 (size=5431) 2024-11-20T20:35:45,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741896_1074 (size=5431) 2024-11-20T20:35:45,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741896_1074 (size=5431) 2024-11-20T20:35:45,347 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/.tmp/table/6b58909dd9d54b91bfecdafd1815baea 2024-11-20T20:35:45,354 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/.tmp/info/58051d9b853d4f248fdf570a1aeb694e as hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/info/58051d9b853d4f248fdf570a1aeb694e 2024-11-20T20:35:45,362 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/info/58051d9b853d4f248fdf570a1aeb694e, entries=36, sequenceid=18, filesize=10.9 K 2024-11-20T20:35:45,364 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/.tmp/ns/6d062c730982413792aa3dae312950e1 as hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/ns/6d062c730982413792aa3dae312950e1 2024-11-20T20:35:45,370 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/ns/6d062c730982413792aa3dae312950e1, entries=2, sequenceid=18, filesize=5.0 K 2024-11-20T20:35:45,371 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/.tmp/table/6b58909dd9d54b91bfecdafd1815baea as hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/table/6b58909dd9d54b91bfecdafd1815baea 2024-11-20T20:35:45,377 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/table/6b58909dd9d54b91bfecdafd1815baea, entries=2, sequenceid=18, filesize=5.3 K 2024-11-20T20:35:45,378 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 557ms, sequenceid=18, compaction requested=false; wal=null 2024-11-20T20:35:45,378 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-20T20:35:45,380 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T20:35:45,380 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T20:35:45,381 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T20:35:45,382 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T20:35:45,385 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-20T20:35:45,386 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61038952, jitterRate=-0.09044873714447021}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T20:35:45,386 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T20:35:45,387 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732134944794Writing region info on filesystem at 1732134944794Initializing all the Stores at 1732134944796 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732134944796Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732134944796Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134944796Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732134944796Obtaining lock to block concurrent updates at 1732134944820 (+24 ms)Preparing flush snapshotting stores in 1588230740 at 1732134944820Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1732134944821 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732134944821Flushing 1588230740/info: creating writer at 1732134944821Flushing 1588230740/info: appending metadata at 1732134944853 (+32 ms)Flushing 1588230740/info: closing flushed file at 1732134944853Flushing 1588230740/ns: creating writer at 1732134945288 (+435 ms)Flushing 1588230740/ns: appending metadata at 1732134945304 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732134945304Flushing 1588230740/table: creating writer at 1732134945317 (+13 ms)Flushing 1588230740/table: appending metadata at 1732134945336 (+19 ms)Flushing 1588230740/table: closing flushed file at 1732134945336Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5746be19: reopening flushed file at 1732134945353 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@409ac1ab: reopening flushed file at 1732134945363 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b1a8549: reopening flushed file at 1732134945370 (+7 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 557ms, sequenceid=18, compaction requested=false; wal=null at 1732134945378 (+8 ms)Cleaning up temporary data from old regions at 1732134945380 (+2 ms)Running coprocessor post-open hooks at 1732134945386 (+6 ms)Region opened successfully at 1732134945387 (+1 ms) 2024-11-20T20:35:45,388 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1732134944750 2024-11-20T20:35:45,390 DEBUG [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T20:35:45,390 INFO [RS_OPEN_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T20:35:45,391 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=d7d59561a64e,33553,1732134919262 2024-11-20T20:35:45,392 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d7d59561a64e,33553,1732134919262, state=OPEN 2024-11-20T20:35:45,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33553-0x10159f54fc60002, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T20:35:45,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36243-0x10159f54fc60003, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T20:35:45,419 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T20:35:45,419 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T20:35:45,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T20:35:45,419 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d7d59561a64e,33553,1732134919262 2024-11-20T20:35:45,419 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T20:35:45,424 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-20T20:35:45,424 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d7d59561a64e,33553,1732134919262 in 832 msec 2024-11-20T20:35:45,429 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-20T20:35:45,429 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.0080 sec 2024-11-20T20:35:45,430 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(207): d7d59561a64e,36639,1732134919021 had 2 regions 2024-11-20T20:35:45,432 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure d7d59561a64e,36639,1732134919021, splitWal=true, meta=true, isMeta: false 2024-11-20T20:35:45,434 INFO [PEWorker-5 {}] master.SplitWALManager(105): d7d59561a64e,36639,1732134919021 WAL count=1, meta=false 2024-11-20T20:35:45,434 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure d7d59561a64e%2C36639%2C1732134919021.1732134922368}] 2024-11-20T20:35:45,437 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=d7d59561a64e,33553,1732134919262 2024-11-20T20:35:45,438 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure d7d59561a64e%2C36639%2C1732134919021.1732134922368, worker=d7d59561a64e,33553,1732134919262}] 2024-11-20T20:35:45,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33553 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-20T20:35:45,609 INFO [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.1732134922368, size=1.6 K (1677bytes) 2024-11-20T20:35:45,609 INFO [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.1732134922368 2024-11-20T20:35:45,610 INFO [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.1732134922368 after 1ms 2024-11-20T20:35:45,612 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.1732134922368: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:45,613 INFO [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.1732134922368 took 3ms 2024-11-20T20:35:45,620 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for e84bafb3668f028684842276e5650d4c: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-20T20:35:45,620 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.1732134922368 so closing down 2024-11-20T20:35:45,620 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T20:35:45,621 INFO [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T20:35:45,621 INFO [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 5 ms; skipped=6; WAL=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.1732134922368, size=1.6 K, length=1677, corrupted=false, cancelled=false 2024-11-20T20:35:45,621 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.1732134922368, journal: Splitting hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.1732134922368, size=1.6 K (1677bytes) at 1732134945609Finishing writing output for hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.1732134922368 so closing down at 1732134945620 (+11 ms)3 split writer threads finished at 1732134945621 (+1 ms)Processed 6 edits across 0 Regions in 5 ms; skipped=6; WAL=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.1732134922368, size=1.6 K, length=1677, corrupted=false, cancelled=false at 1732134945621 2024-11-20T20:35:45,621 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.1732134922368 2024-11-20T20:35:45,621 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d7d59561a64e:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-20T20:35:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46641 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-20T20:35:45,631 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting/d7d59561a64e%2C36639%2C1732134919021.1732134922368 to hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/oldWALs 2024-11-20T20:35:45,640 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-20T20:35:45,640 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d7d59561a64e%2C36639%2C1732134919021.1732134922368, worker=d7d59561a64e,33553,1732134919262 in 193 msec 2024-11-20T20:35:45,644 DEBUG [PEWorker-4 {}] master.SplitWALManager(172): Release split WAL worker=d7d59561a64e,33553,1732134919262 2024-11-20T20:35:45,652 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-20T20:35:45,652 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure d7d59561a64e%2C36639%2C1732134919021.1732134922368, worker=d7d59561a64e,33553,1732134919262 in 210 msec 2024-11-20T20:35:45,654 INFO [PEWorker-5 {}] master.SplitLogManager(171): hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,36639,1732134919021-splitting dir is empty, no logs to split. 2024-11-20T20:35:45,654 INFO [PEWorker-5 {}] master.SplitWALManager(105): d7d59561a64e,36639,1732134919021 WAL count=0, meta=false 2024-11-20T20:35:45,654 DEBUG [PEWorker-5 {}] procedure.ServerCrashProcedure(329): Check if d7d59561a64e,36639,1732134919021 WAL splitting is done? wals=0, meta=false 2024-11-20T20:35:45,657 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e84bafb3668f028684842276e5650d4c, ASSIGN}] 2024-11-20T20:35:45,659 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e84bafb3668f028684842276e5650d4c, ASSIGN 2024-11-20T20:35:45,662 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e84bafb3668f028684842276e5650d4c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T20:35:45,812 DEBUG [d7d59561a64e:46641 {}] balancer.BalancerClusterState(204): Hosts are {d7d59561a64e=0} racks are {/default-rack=0} 2024-11-20T20:35:45,813 DEBUG [d7d59561a64e:46641 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T20:35:45,813 DEBUG [d7d59561a64e:46641 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T20:35:45,813 DEBUG [d7d59561a64e:46641 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T20:35:45,813 DEBUG [d7d59561a64e:46641 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T20:35:45,813 INFO [d7d59561a64e:46641 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T20:35:45,813 INFO [d7d59561a64e:46641 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T20:35:45,813 DEBUG [d7d59561a64e:46641 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T20:35:45,813 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=e84bafb3668f028684842276e5650d4c, regionState=OPENING, regionLocation=d7d59561a64e,36243,1732134919381 2024-11-20T20:35:45,814 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server d7d59561a64e:36639 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: d7d59561a64e/172.17.0.2:36639 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:35:45,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,36639,1732134919021, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,36639,1732134919021, seqNum=-1, error=java.net.ConnectException: Call to address=d7d59561a64e:36639 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: d7d59561a64e/172.17.0.2:36639 2024-11-20T20:35:45,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,36639,1732134919021, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-20T20:35:45,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,36639,1732134919021, seqNum=-1 from cache 2024-11-20T20:35:45,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address d7d59561a64e:36639 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: d7d59561a64e/172.17.0.2:36639 2024-11-20T20:35:45,821 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T20:35:45,822 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,33553,1732134919262, seqNum=-1] 2024-11-20T20:35:45,920 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T20:35:45,920 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d7d59561a64e,33553,1732134919262, seqNum=-1] 2024-11-20T20:35:45,921 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T20:35:45,923 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41959, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T20:35:45,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e84bafb3668f028684842276e5650d4c, ASSIGN because future has completed 2024-11-20T20:35:45,929 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure e84bafb3668f028684842276e5650d4c, server=d7d59561a64e,36243,1732134919381}] 2024-11-20T20:35:45,940 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to d7d59561a64e:36639 this server is in the failed servers list 2024-11-20T20:35:45,941 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., hostname=d7d59561a64e,36639,1732134919021, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., hostname=d7d59561a64e,36639,1732134919021, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=d7d59561a64e:36639 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: d7d59561a64e:36639 2024-11-20T20:35:45,941 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., hostname=d7d59561a64e,36639,1732134919021, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: d7d59561a64e:36639 2024-11-20T20:35:45,941 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., hostname=d7d59561a64e,36639,1732134919021, seqNum=12 from cache 2024-11-20T20:35:46,090 INFO [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:46,090 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => e84bafb3668f028684842276e5650d4c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c.', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:35:46,090 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:46,091 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:46,091 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:46,091 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:46,094 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:46,096 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e84bafb3668f028684842276e5650d4c columnFamilyName cf1 2024-11-20T20:35:46,096 DEBUG [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:46,108 DEBUG [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf1/abd38df55e1c456c90a12675eeae07e2 2024-11-20T20:35:46,108 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.HStore(327): Store=e84bafb3668f028684842276e5650d4c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:46,108 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:46,110 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e84bafb3668f028684842276e5650d4c columnFamilyName cf2 2024-11-20T20:35:46,110 DEBUG [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:46,127 DEBUG [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/cf2/c63971d5759c499faadf4cf40b2962a5 2024-11-20T20:35:46,127 INFO [StoreOpener-e84bafb3668f028684842276e5650d4c-1 {}] regionserver.HStore(327): Store=e84bafb3668f028684842276e5650d4c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:46,128 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:46,129 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:46,131 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:46,132 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:46,132 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:46,133 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-20T20:35:46,135 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:46,136 INFO [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened e84bafb3668f028684842276e5650d4c; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67471669, jitterRate=0.005406215786933899}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-20T20:35:46,136 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e84bafb3668f028684842276e5650d4c 2024-11-20T20:35:46,137 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for e84bafb3668f028684842276e5650d4c: Running coprocessor pre-open hook at 1732134946091Writing region info on filesystem at 1732134946091Initializing all the Stores at 1732134946092 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134946093 (+1 ms)Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134946093Cleaning up temporary data from old regions at 1732134946132 (+39 ms)Running coprocessor post-open hooks at 1732134946136 (+4 ms)Region opened successfully at 1732134946137 (+1 ms) 2024-11-20T20:35:46,141 INFO [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., pid=21, masterSystemTime=1732134946084 2024-11-20T20:35:46,150 DEBUG [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:46,150 INFO [RS_OPEN_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:35:46,160 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=e84bafb3668f028684842276e5650d4c, regionState=OPEN, openSeqNum=18, regionLocation=d7d59561a64e,36243,1732134919381 2024-11-20T20:35:46,161 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., hostname=d7d59561a64e,36639,1732134919021, seqNum=18] 2024-11-20T20:35:46,162 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server d7d59561a64e:36639 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: d7d59561a64e/172.17.0.2:36639 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:35:46,163 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., hostname=d7d59561a64e,36639,1732134919021, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., hostname=d7d59561a64e,36639,1732134919021, seqNum=18, error=java.net.ConnectException: Call to address=d7d59561a64e:36639 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: d7d59561a64e/172.17.0.2:36639 2024-11-20T20:35:46,163 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., hostname=d7d59561a64e,36639,1732134919021, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-20T20:35:46,163 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., hostname=d7d59561a64e,36639,1732134919021, seqNum=18 from cache 2024-11-20T20:35:46,163 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address d7d59561a64e:36639 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: d7d59561a64e/172.17.0.2:36639 2024-11-20T20:35:46,178 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure e84bafb3668f028684842276e5650d4c, server=d7d59561a64e,36243,1732134919381 because future has completed 2024-11-20T20:35:46,235 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-20T20:35:46,235 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure e84bafb3668f028684842276e5650d4c, server=d7d59561a64e,36243,1732134919381 in 263 msec 2024-11-20T20:35:46,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-20T20:35:46,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=e84bafb3668f028684842276e5650d4c, ASSIGN in 579 msec 2024-11-20T20:35:46,277 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(291): removed crashed server d7d59561a64e,36639,1732134919021 after splitting done 2024-11-20T20:35:46,301 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure d7d59561a64e,36639,1732134919021, splitWal=true, meta=true in 2.3310 sec 2024-11-20T20:35:46,485 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c., hostname=d7d59561a64e,36243,1732134919381, seqNum=18] 2024-11-20T20:35:46,500 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T20:35:46,516 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T20:35:46,549 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=404 (was 411), OpenFileDescriptor=1040 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=933 (was 922) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1575 (was 1729) 2024-11-20T20:35:46,550 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1040 is superior to 1024 2024-11-20T20:35:46,564 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=404, OpenFileDescriptor=1040, MaxFileDescriptor=1048576, SystemLoadAverage=933, ProcessCount=11, AvailableMemoryMB=1571 2024-11-20T20:35:46,565 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1040 is superior to 1024 2024-11-20T20:35:46,587 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:46,590 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:46,591 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T20:35:46,595 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-05582734, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/hregion-05582734, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:46,611 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-05582734/hregion-05582734.1732134946595, exclude list is [], retry=0 2024-11-20T20:35:46,614 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:46,614 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:46,615 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:46,620 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-05582734/hregion-05582734.1732134946595 2024-11-20T20:35:46,624 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38233:38233),(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:39759:39759)] 2024-11-20T20:35:46,624 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9c10148c445806ab9842d16255c116b3, NAME => 'testReplayEditsWrittenViaHRegion,,1732134946588.9c10148c445806ab9842d16255c116b3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33605/hbase 2024-11-20T20:35:46,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741898_1076 (size=67) 2024-11-20T20:35:46,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741898_1076 (size=67) 2024-11-20T20:35:46,644 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732134946588.9c10148c445806ab9842d16255c116b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:46,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741898_1076 (size=67) 2024-11-20T20:35:46,672 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:46,693 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c10148c445806ab9842d16255c116b3 columnFamilyName a 2024-11-20T20:35:46,693 DEBUG [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:46,695 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] regionserver.HStore(327): Store=9c10148c445806ab9842d16255c116b3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:46,696 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:46,698 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c10148c445806ab9842d16255c116b3 columnFamilyName b 2024-11-20T20:35:46,698 DEBUG [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:46,699 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] regionserver.HStore(327): Store=9c10148c445806ab9842d16255c116b3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:46,699 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:46,703 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c10148c445806ab9842d16255c116b3 columnFamilyName c 2024-11-20T20:35:46,703 DEBUG [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:46,708 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] regionserver.HStore(327): Store=9c10148c445806ab9842d16255c116b3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:46,708 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:46,709 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:46,710 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:46,712 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:46,712 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:46,716 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T20:35:46,724 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:46,734 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T20:35:46,735 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9c10148c445806ab9842d16255c116b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70588949, jitterRate=0.05185730755329132}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T20:35:46,735 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9c10148c445806ab9842d16255c116b3: Writing region info on filesystem at 1732134946644Initializing all the Stores at 1732134946664 (+20 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134946664Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134946668 (+4 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134946668Cleaning up temporary data from old regions at 1732134946712 (+44 ms)Region opened successfully at 1732134946735 (+23 ms) 2024-11-20T20:35:46,735 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9c10148c445806ab9842d16255c116b3, disabling compactions & flushes 2024-11-20T20:35:46,735 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732134946588.9c10148c445806ab9842d16255c116b3. 2024-11-20T20:35:46,735 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732134946588.9c10148c445806ab9842d16255c116b3. 2024-11-20T20:35:46,735 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732134946588.9c10148c445806ab9842d16255c116b3. after waiting 0 ms 2024-11-20T20:35:46,736 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732134946588.9c10148c445806ab9842d16255c116b3. 2024-11-20T20:35:46,736 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732134946588.9c10148c445806ab9842d16255c116b3. 2024-11-20T20:35:46,736 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9c10148c445806ab9842d16255c116b3: Waiting for close lock at 1732134946735Disabling compacts and flushes for region at 1732134946735Disabling writes for close at 1732134946735Writing region close event to WAL at 1732134946736 (+1 ms)Closed at 1732134946736 2024-11-20T20:35:46,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741897_1075 (size=93) 2024-11-20T20:35:46,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741897_1075 (size=93) 2024-11-20T20:35:46,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741897_1075 (size=93) 2024-11-20T20:35:46,761 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T20:35:46,761 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-05582734:(num 1732134946595) 2024-11-20T20:35:46,762 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T20:35:46,764 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732134946586, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:46,791 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732134946586/wal.1732134946764, exclude list is [], retry=0 2024-11-20T20:35:46,794 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:46,794 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:46,794 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:46,799 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T20:35:46,816 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732134946586/wal.1732134946764 2024-11-20T20:35:46,822 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39759:39759),(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:38233:38233)] 2024-11-20T20:35:46,822 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9c10148c445806ab9842d16255c116b3, NAME => 'testReplayEditsWrittenViaHRegion,,1732134946588.9c10148c445806ab9842d16255c116b3.', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:35:46,824 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732134946588.9c10148c445806ab9842d16255c116b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:46,824 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:46,824 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:46,853 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:46,856 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c10148c445806ab9842d16255c116b3 columnFamilyName a 2024-11-20T20:35:46,856 DEBUG [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:46,865 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] regionserver.HStore(327): Store=9c10148c445806ab9842d16255c116b3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:46,865 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:46,866 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c10148c445806ab9842d16255c116b3 columnFamilyName b 2024-11-20T20:35:46,866 DEBUG [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:46,867 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] regionserver.HStore(327): Store=9c10148c445806ab9842d16255c116b3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:46,867 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:46,868 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c10148c445806ab9842d16255c116b3 columnFamilyName c 2024-11-20T20:35:46,875 DEBUG [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:46,876 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] regionserver.HStore(327): Store=9c10148c445806ab9842d16255c116b3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:46,876 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:46,881 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:46,883 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:46,885 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:46,886 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:46,886 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T20:35:46,897 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:46,899 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9c10148c445806ab9842d16255c116b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65068170, jitterRate=-0.03040871024131775}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T20:35:46,901 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9c10148c445806ab9842d16255c116b3: Writing region info on filesystem at 1732134946831Initializing all the Stores at 1732134946852 (+21 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134946853 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134946853Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134946853Cleaning up temporary data from old regions at 1732134946886 (+33 ms)Region opened successfully at 1732134946901 (+15 ms) 2024-11-20T20:35:47,013 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9c10148c445806ab9842d16255c116b3 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-20T20:35:47,045 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/.tmp/a/b5c01919538a4643b8bf7f7af2a730c4 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732134946901/Put/seqid=0 2024-11-20T20:35:47,090 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T20:35:47,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741900_1078 (size=5958) 2024-11-20T20:35:47,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741900_1078 (size=5958) 2024-11-20T20:35:47,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741900_1078 (size=5958) 2024-11-20T20:35:47,099 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/.tmp/a/b5c01919538a4643b8bf7f7af2a730c4 2024-11-20T20:35:47,122 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/.tmp/b/6963e18f414a439cb78a7e6f766e7600 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732134946954/Put/seqid=0 2024-11-20T20:35:47,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741901_1079 (size=5958) 2024-11-20T20:35:47,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741901_1079 (size=5958) 2024-11-20T20:35:47,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741901_1079 (size=5958) 2024-11-20T20:35:47,548 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/.tmp/b/6963e18f414a439cb78a7e6f766e7600 2024-11-20T20:35:47,572 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/.tmp/c/6057333b6c634efabaf7652c45decc80 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732134946980/Put/seqid=0 2024-11-20T20:35:47,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741902_1080 (size=5958) 2024-11-20T20:35:47,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741902_1080 (size=5958) 2024-11-20T20:35:47,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741902_1080 (size=5958) 2024-11-20T20:35:47,580 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/.tmp/c/6057333b6c634efabaf7652c45decc80 2024-11-20T20:35:47,586 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/.tmp/a/b5c01919538a4643b8bf7f7af2a730c4 as hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/a/b5c01919538a4643b8bf7f7af2a730c4 2024-11-20T20:35:47,591 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/a/b5c01919538a4643b8bf7f7af2a730c4, entries=10, sequenceid=33, filesize=5.8 K 2024-11-20T20:35:47,593 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/.tmp/b/6963e18f414a439cb78a7e6f766e7600 as hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/b/6963e18f414a439cb78a7e6f766e7600 2024-11-20T20:35:47,598 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/b/6963e18f414a439cb78a7e6f766e7600, entries=10, sequenceid=33, filesize=5.8 K 2024-11-20T20:35:47,599 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/.tmp/c/6057333b6c634efabaf7652c45decc80 as hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/c/6057333b6c634efabaf7652c45decc80 2024-11-20T20:35:47,606 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/c/6057333b6c634efabaf7652c45decc80, entries=10, sequenceid=33, filesize=5.8 K 2024-11-20T20:35:47,607 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 9c10148c445806ab9842d16255c116b3 in 595ms, sequenceid=33, compaction requested=false 2024-11-20T20:35:47,607 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9c10148c445806ab9842d16255c116b3: 2024-11-20T20:35:47,608 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9c10148c445806ab9842d16255c116b3, disabling compactions & flushes 2024-11-20T20:35:47,608 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732134946588.9c10148c445806ab9842d16255c116b3. 2024-11-20T20:35:47,608 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732134946588.9c10148c445806ab9842d16255c116b3. 2024-11-20T20:35:47,608 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732134946588.9c10148c445806ab9842d16255c116b3. after waiting 0 ms 2024-11-20T20:35:47,608 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732134946588.9c10148c445806ab9842d16255c116b3. 2024-11-20T20:35:47,609 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732134946588.9c10148c445806ab9842d16255c116b3. 2024-11-20T20:35:47,609 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9c10148c445806ab9842d16255c116b3: Waiting for close lock at 1732134947608Disabling compacts and flushes for region at 1732134947608Disabling writes for close at 1732134947608Writing region close event to WAL at 1732134947609 (+1 ms)Closed at 1732134947609 2024-11-20T20:35:47,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741899_1077 (size=2873) 2024-11-20T20:35:47,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741899_1077 (size=2873) 2024-11-20T20:35:47,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741899_1077 (size=2873) 2024-11-20T20:35:47,617 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/b/6963e18f414a439cb78a7e6f766e7600 to hdfs://localhost:33605/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/b/6963e18f414a439cb78a7e6f766e7600 2024-11-20T20:35:47,633 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33605/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732134946586/wal.1732134946764, size=2.8 K (2873bytes) 2024-11-20T20:35:47,633 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33605/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732134946586/wal.1732134946764 2024-11-20T20:35:47,635 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33605/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732134946586/wal.1732134946764 after 2ms 2024-11-20T20:35:47,638 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732134946586/wal.1732134946764: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:47,638 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33605/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732134946586/wal.1732134946764 took 5ms 2024-11-20T20:35:47,643 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33605/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732134946586/wal.1732134946764 so closing down 2024-11-20T20:35:47,643 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T20:35:47,650 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732134946764.temp 2024-11-20T20:35:47,651 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/recovered.edits/0000000000000000003-wal.1732134946764.temp 2024-11-20T20:35:47,652 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T20:35:47,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741903_1081 (size=2312) 2024-11-20T20:35:47,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741903_1081 (size=2312) 2024-11-20T20:35:47,669 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/recovered.edits/0000000000000000003-wal.1732134946764.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-20T20:35:47,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741903_1081 (size=2312) 2024-11-20T20:35:47,670 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/recovered.edits/0000000000000000003-wal.1732134946764.temp to hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/recovered.edits/0000000000000000032 2024-11-20T20:35:47,670 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 29 ms; skipped=2; WAL=hdfs://localhost:33605/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732134946586/wal.1732134946764, size=2.8 K, length=2873, corrupted=false, cancelled=false 2024-11-20T20:35:47,671 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33605/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732134946586/wal.1732134946764, journal: Splitting hdfs://localhost:33605/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732134946586/wal.1732134946764, size=2.8 K (2873bytes) at 1732134947633Finishing writing output for hdfs://localhost:33605/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732134946586/wal.1732134946764 so closing down at 1732134947643 (+10 ms)Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/recovered.edits/0000000000000000003-wal.1732134946764.temp at 1732134947651 (+8 ms)3 split writer threads finished at 1732134947652 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/recovered.edits/0000000000000000003-wal.1732134946764.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732134947669 (+17 ms)Rename recovered edits hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/recovered.edits/0000000000000000003-wal.1732134946764.temp to hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/recovered.edits/0000000000000000032 at 1732134947670 (+1 ms)Processed 32 edits across 1 Regions in 29 ms; skipped=2; WAL=hdfs://localhost:33605/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732134946586/wal.1732134946764, size=2.8 K, length=2873, corrupted=false, cancelled=false at 1732134947670 2024-11-20T20:35:47,672 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33605/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732134946586/wal.1732134946764 to hdfs://localhost:33605/hbase/oldWALs/wal.1732134946764 2024-11-20T20:35:47,673 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/recovered.edits/0000000000000000032 2024-11-20T20:35:47,673 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T20:35:47,675 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732134946586, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:47,688 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732134946586/wal.1732134947675, exclude list is [], retry=0 2024-11-20T20:35:47,692 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:47,692 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:47,693 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:47,701 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732134946586/wal.1732134947675 2024-11-20T20:35:47,704 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39759:39759),(127.0.0.1/127.0.0.1:38233:38233),(127.0.0.1/127.0.0.1:46511:46511)] 2024-11-20T20:35:47,704 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9c10148c445806ab9842d16255c116b3, NAME => 'testReplayEditsWrittenViaHRegion,,1732134946588.9c10148c445806ab9842d16255c116b3.', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:35:47,705 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732134946588.9c10148c445806ab9842d16255c116b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:47,705 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:47,705 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:47,708 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:47,709 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c10148c445806ab9842d16255c116b3 columnFamilyName a 2024-11-20T20:35:47,709 DEBUG [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:47,717 DEBUG [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/a/b5c01919538a4643b8bf7f7af2a730c4 2024-11-20T20:35:47,717 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] regionserver.HStore(327): Store=9c10148c445806ab9842d16255c116b3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:47,718 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:47,720 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c10148c445806ab9842d16255c116b3 columnFamilyName b 2024-11-20T20:35:47,721 DEBUG [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:47,725 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] regionserver.HStore(327): Store=9c10148c445806ab9842d16255c116b3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:47,725 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:47,726 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c10148c445806ab9842d16255c116b3 columnFamilyName c 2024-11-20T20:35:47,726 DEBUG [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:47,739 DEBUG [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/c/6057333b6c634efabaf7652c45decc80 2024-11-20T20:35:47,739 INFO [StoreOpener-9c10148c445806ab9842d16255c116b3-1 {}] regionserver.HStore(327): Store=9c10148c445806ab9842d16255c116b3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:47,740 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:47,741 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:47,742 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:47,744 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/recovered.edits/0000000000000000032 2024-11-20T20:35:47,748 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:47,749 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/recovered.edits/0000000000000000032 2024-11-20T20:35:47,749 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9c10148c445806ab9842d16255c116b3 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-20T20:35:47,782 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/.tmp/b/d5cdb7259cdb4485b9d4d17f24543d9d is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732134946954/Put/seqid=0 2024-11-20T20:35:47,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741905_1083 (size=5958) 2024-11-20T20:35:47,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741905_1083 (size=5958) 2024-11-20T20:35:47,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741905_1083 (size=5958) 2024-11-20T20:35:47,805 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/.tmp/b/d5cdb7259cdb4485b9d4d17f24543d9d 2024-11-20T20:35:47,815 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/.tmp/b/d5cdb7259cdb4485b9d4d17f24543d9d as hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/b/d5cdb7259cdb4485b9d4d17f24543d9d 2024-11-20T20:35:47,836 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/b/d5cdb7259cdb4485b9d4d17f24543d9d, entries=10, sequenceid=32, filesize=5.8 K 2024-11-20T20:35:47,836 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 9c10148c445806ab9842d16255c116b3 in 87ms, sequenceid=32, compaction requested=false; wal=null 2024-11-20T20:35:47,837 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/recovered.edits/0000000000000000032 2024-11-20T20:35:47,860 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:47,860 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:47,862 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T20:35:47,865 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9c10148c445806ab9842d16255c116b3 2024-11-20T20:35:47,891 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/9c10148c445806ab9842d16255c116b3/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-20T20:35:47,896 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9c10148c445806ab9842d16255c116b3; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68471093, jitterRate=0.0202987939119339}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T20:35:47,897 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9c10148c445806ab9842d16255c116b3: Writing region info on filesystem at 1732134947705Initializing all the Stores at 1732134947707 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134947707Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134947708 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134947708Obtaining lock to block concurrent updates at 1732134947750 (+42 ms)Preparing flush snapshotting stores in 9c10148c445806ab9842d16255c116b3 at 1732134947750Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732134946588.9c10148c445806ab9842d16255c116b3., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732134947750Flushing stores of testReplayEditsWrittenViaHRegion,,1732134946588.9c10148c445806ab9842d16255c116b3. at 1732134947750Flushing 9c10148c445806ab9842d16255c116b3/b: creating writer at 1732134947750Flushing 9c10148c445806ab9842d16255c116b3/b: appending metadata at 1732134947781 (+31 ms)Flushing 9c10148c445806ab9842d16255c116b3/b: closing flushed file at 1732134947781Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b7bc66b: reopening flushed file at 1732134947814 (+33 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 9c10148c445806ab9842d16255c116b3 in 87ms, sequenceid=32, compaction requested=false; wal=null at 1732134947836 (+22 ms)Cleaning up temporary data from old regions at 1732134947860 (+24 ms)Region opened successfully at 1732134947897 (+37 ms) 2024-11-20T20:35:47,934 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=414 (was 404) Potentially hanging thread: PacketResponder: BP-1180650432-172.17.0.2-1732134912528:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:60378 [Waiting for operation #44] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:35808 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:58636 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:59470 [Waiting for operation #43] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:42284 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1180650432-172.17.0.2-1732134912528:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:45480 [Waiting for operation #36] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1180650432-172.17.0.2-1732134912528:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1116 (was 1040) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=933 (was 933), ProcessCount=11 (was 11), AvailableMemoryMB=1561 (was 1571) 2024-11-20T20:35:47,934 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1116 is superior to 1024 2024-11-20T20:35:47,948 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=414, OpenFileDescriptor=1116, MaxFileDescriptor=1048576, SystemLoadAverage=933, ProcessCount=11, AvailableMemoryMB=1559 2024-11-20T20:35:47,948 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1116 is superior to 1024 2024-11-20T20:35:47,962 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:47,964 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:47,965 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T20:35:47,967 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-59612681, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/hregion-59612681, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:47,980 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-59612681/hregion-59612681.1732134947967, exclude list is [], retry=0 2024-11-20T20:35:47,983 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:47,983 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:47,983 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:47,988 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-59612681/hregion-59612681.1732134947967 2024-11-20T20:35:47,988 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:38233:38233),(127.0.0.1/127.0.0.1:39759:39759)] 2024-11-20T20:35:47,989 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e188d891e51b57da9db6493440046b4b, NAME => 'testReplayEditsAfterAbortingFlush,,1732134947963.e188d891e51b57da9db6493440046b4b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33605/hbase 2024-11-20T20:35:48,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741907_1085 (size=68) 2024-11-20T20:35:48,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741907_1085 (size=68) 2024-11-20T20:35:48,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741907_1085 (size=68) 2024-11-20T20:35:48,008 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732134947963.e188d891e51b57da9db6493440046b4b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:48,012 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:48,015 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e188d891e51b57da9db6493440046b4b columnFamilyName a 2024-11-20T20:35:48,015 DEBUG [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:48,016 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] regionserver.HStore(327): Store=e188d891e51b57da9db6493440046b4b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:48,016 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:48,018 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e188d891e51b57da9db6493440046b4b columnFamilyName b 2024-11-20T20:35:48,018 DEBUG [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:48,018 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] regionserver.HStore(327): Store=e188d891e51b57da9db6493440046b4b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:48,019 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:48,020 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e188d891e51b57da9db6493440046b4b columnFamilyName c 2024-11-20T20:35:48,021 DEBUG [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:48,022 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] regionserver.HStore(327): Store=e188d891e51b57da9db6493440046b4b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:48,022 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:48,023 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:48,023 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:48,024 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:48,024 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:48,025 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T20:35:48,027 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:48,037 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T20:35:48,038 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e188d891e51b57da9db6493440046b4b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63871090, jitterRate=-0.0482465922832489}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T20:35:48,039 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e188d891e51b57da9db6493440046b4b: Writing region info on filesystem at 1732134948008Initializing all the Stores at 1732134948009 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134948009Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134948012 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134948012Cleaning up temporary data from old regions at 1732134948024 (+12 ms)Region opened successfully at 1732134948038 (+14 ms) 2024-11-20T20:35:48,039 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e188d891e51b57da9db6493440046b4b, disabling compactions & flushes 2024-11-20T20:35:48,039 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732134947963.e188d891e51b57da9db6493440046b4b. 2024-11-20T20:35:48,039 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732134947963.e188d891e51b57da9db6493440046b4b. 2024-11-20T20:35:48,039 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732134947963.e188d891e51b57da9db6493440046b4b. after waiting 0 ms 2024-11-20T20:35:48,039 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732134947963.e188d891e51b57da9db6493440046b4b. 2024-11-20T20:35:48,040 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732134947963.e188d891e51b57da9db6493440046b4b. 2024-11-20T20:35:48,041 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e188d891e51b57da9db6493440046b4b: Waiting for close lock at 1732134948039Disabling compacts and flushes for region at 1732134948039Disabling writes for close at 1732134948039Writing region close event to WAL at 1732134948040 (+1 ms)Closed at 1732134948040 2024-11-20T20:35:48,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741906_1084 (size=93) 2024-11-20T20:35:48,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741906_1084 (size=93) 2024-11-20T20:35:48,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741906_1084 (size=93) 2024-11-20T20:35:48,054 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T20:35:48,054 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-59612681:(num 1732134947967) 2024-11-20T20:35:48,055 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T20:35:48,058 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732134947961, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:48,070 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732134947961/wal.1732134948058, exclude list is [], retry=0 2024-11-20T20:35:48,073 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:48,073 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:48,073 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:48,076 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732134947961/wal.1732134948058 2024-11-20T20:35:48,078 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:39759:39759),(127.0.0.1/127.0.0.1:38233:38233)] 2024-11-20T20:35:48,184 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e188d891e51b57da9db6493440046b4b, NAME => 'testReplayEditsAfterAbortingFlush,,1732134947963.e188d891e51b57da9db6493440046b4b.', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:35:48,189 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:48,189 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732134947963.e188d891e51b57da9db6493440046b4b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:48,189 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:48,189 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:48,205 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:48,207 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e188d891e51b57da9db6493440046b4b columnFamilyName a 2024-11-20T20:35:48,208 DEBUG [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:48,209 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] regionserver.HStore(327): Store=e188d891e51b57da9db6493440046b4b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:48,209 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:48,210 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e188d891e51b57da9db6493440046b4b columnFamilyName b 2024-11-20T20:35:48,210 DEBUG [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:48,211 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] regionserver.HStore(327): Store=e188d891e51b57da9db6493440046b4b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:48,211 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:48,219 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e188d891e51b57da9db6493440046b4b columnFamilyName c 2024-11-20T20:35:48,219 DEBUG [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:48,225 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] regionserver.HStore(327): Store=e188d891e51b57da9db6493440046b4b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:48,225 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:48,226 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:48,233 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:48,237 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:48,237 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:48,238 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T20:35:48,239 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:48,241 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e188d891e51b57da9db6493440046b4b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74612356, jitterRate=0.11181074380874634}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T20:35:48,243 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:48,245 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e188d891e51b57da9db6493440046b4b: Running coprocessor pre-open hook at 1732134948190Writing region info on filesystem at 1732134948190Initializing all the Stores at 1732134948200 (+10 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134948200Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134948204 (+4 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134948204Cleaning up temporary data from old regions at 1732134948237 (+33 ms)Running coprocessor post-open hooks at 1732134948244 (+7 ms)Region opened successfully at 1732134948244 2024-11-20T20:35:48,315 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e188d891e51b57da9db6493440046b4b 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-20T20:35:48,329 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e188d891e51b57da9db6493440046b4b/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:35:48,415 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-20T20:35:48,416 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-20T20:35:48,417 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-20T20:35:48,417 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-20T20:35:48,418 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-20T20:35:48,418 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-20T20:35:49,340 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e188d891e51b57da9db6493440046b4b/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:35:50,341 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e188d891e51b57da9db6493440046b4b/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:35:51,342 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e188d891e51b57da9db6493440046b4b/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:35:52,343 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e188d891e51b57da9db6493440046b4b/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:35:53,343 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e188d891e51b57da9db6493440046b4b/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:35:53,692 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T20:35:54,344 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e188d891e51b57da9db6493440046b4b/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:35:55,345 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e188d891e51b57da9db6493440046b4b/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:35:56,345 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e188d891e51b57da9db6493440046b4b/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:35:57,346 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e188d891e51b57da9db6493440046b4b/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:35:57,347 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e188d891e51b57da9db6493440046b4b: 2024-11-20T20:35:57,347 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732134947963.e188d891e51b57da9db6493440046b4b. 2024-11-20T20:35:57,359 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e188d891e51b57da9db6493440046b4b: 2024-11-20T20:35:57,359 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-20T20:35:57,359 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e188d891e51b57da9db6493440046b4b, disabling compactions & flushes 2024-11-20T20:35:57,359 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732134947963.e188d891e51b57da9db6493440046b4b. 2024-11-20T20:35:57,359 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732134947963.e188d891e51b57da9db6493440046b4b. 2024-11-20T20:35:57,359 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732134947963.e188d891e51b57da9db6493440046b4b. after waiting 0 ms 2024-11-20T20:35:57,359 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732134947963.e188d891e51b57da9db6493440046b4b. 2024-11-20T20:35:57,360 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732134947963.e188d891e51b57da9db6493440046b4b. 2024-11-20T20:35:57,360 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732134947963.e188d891e51b57da9db6493440046b4b. 2024-11-20T20:35:57,360 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e188d891e51b57da9db6493440046b4b: Waiting for close lock at 1732134957359Running coprocessor pre-close hooks at 1732134957359Disabling compacts and flushes for region at 1732134957359Disabling writes for close at 1732134957359Writing region close event to WAL at 1732134957360 (+1 ms)Running coprocessor post-close hooks at 1732134957360Closed at 1732134957360 2024-11-20T20:35:57,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741908_1086 (size=2398) 2024-11-20T20:35:57,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741908_1086 (size=2398) 2024-11-20T20:35:57,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741908_1086 (size=2398) 2024-11-20T20:35:57,382 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33605/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732134947961/wal.1732134948058, size=2.3 K (2398bytes) 2024-11-20T20:35:57,382 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33605/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732134947961/wal.1732134948058 2024-11-20T20:35:57,383 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33605/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732134947961/wal.1732134948058 after 1ms 2024-11-20T20:35:57,385 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732134947961/wal.1732134948058: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:57,385 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33605/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732134947961/wal.1732134948058 took 3ms 2024-11-20T20:35:57,388 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33605/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732134947961/wal.1732134948058 so closing down 2024-11-20T20:35:57,388 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T20:35:57,389 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732134948058.temp 2024-11-20T20:35:57,390 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/recovered.edits/0000000000000000004-wal.1732134948058.temp 2024-11-20T20:35:57,390 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T20:35:57,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741909_1087 (size=1672) 2024-11-20T20:35:57,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741909_1087 (size=1672) 2024-11-20T20:35:57,397 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/recovered.edits/0000000000000000004-wal.1732134948058.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-20T20:35:57,399 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/recovered.edits/0000000000000000004-wal.1732134948058.temp to hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/recovered.edits/0000000000000000026 2024-11-20T20:35:57,399 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:33605/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732134947961/wal.1732134948058, size=2.3 K, length=2398, corrupted=false, cancelled=false 2024-11-20T20:35:57,399 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33605/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732134947961/wal.1732134948058, journal: Splitting hdfs://localhost:33605/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732134947961/wal.1732134948058, size=2.3 K (2398bytes) at 1732134957382Finishing writing output for hdfs://localhost:33605/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732134947961/wal.1732134948058 so closing down at 1732134957388 (+6 ms)Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/recovered.edits/0000000000000000004-wal.1732134948058.temp at 1732134957390 (+2 ms)3 split writer threads finished at 1732134957390Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/recovered.edits/0000000000000000004-wal.1732134948058.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732134957397 (+7 ms)Rename recovered edits hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/recovered.edits/0000000000000000004-wal.1732134948058.temp to hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/recovered.edits/0000000000000000026 at 1732134957399 (+2 ms)Processed 23 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:33605/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732134947961/wal.1732134948058, size=2.3 K, length=2398, corrupted=false, cancelled=false at 1732134957399 2024-11-20T20:35:57,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741909_1087 (size=1672) 2024-11-20T20:35:57,402 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33605/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732134947961/wal.1732134948058 to hdfs://localhost:33605/hbase/oldWALs/wal.1732134948058 2024-11-20T20:35:57,403 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/recovered.edits/0000000000000000026 2024-11-20T20:35:57,403 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T20:35:57,404 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732134947961, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:57,417 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732134947961/wal.1732134957405, exclude list is [], retry=0 2024-11-20T20:35:57,419 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:57,420 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:57,420 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:57,422 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732134947961/wal.1732134957405 2024-11-20T20:35:57,422 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:38233:38233),(127.0.0.1/127.0.0.1:39759:39759)] 2024-11-20T20:35:57,422 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e188d891e51b57da9db6493440046b4b, NAME => 'testReplayEditsAfterAbortingFlush,,1732134947963.e188d891e51b57da9db6493440046b4b.', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:35:57,423 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:57,423 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732134947963.e188d891e51b57da9db6493440046b4b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:57,423 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:57,423 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:57,425 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:57,426 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e188d891e51b57da9db6493440046b4b columnFamilyName a 2024-11-20T20:35:57,426 DEBUG [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:57,426 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] regionserver.HStore(327): Store=e188d891e51b57da9db6493440046b4b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:57,427 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:57,427 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e188d891e51b57da9db6493440046b4b columnFamilyName b 2024-11-20T20:35:57,427 DEBUG [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:57,428 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] regionserver.HStore(327): Store=e188d891e51b57da9db6493440046b4b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:57,428 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:57,429 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e188d891e51b57da9db6493440046b4b columnFamilyName c 2024-11-20T20:35:57,429 DEBUG [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:57,429 INFO [StoreOpener-e188d891e51b57da9db6493440046b4b-1 {}] regionserver.HStore(327): Store=e188d891e51b57da9db6493440046b4b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:57,429 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:57,430 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:57,431 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:57,432 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/recovered.edits/0000000000000000026 2024-11-20T20:35:57,434 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:57,435 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/recovered.edits/0000000000000000026 2024-11-20T20:35:57,435 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e188d891e51b57da9db6493440046b4b 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-20T20:35:57,450 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/.tmp/a/ca33b1cde15543929ef6098e4da6fb65 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732134957350/Put/seqid=0 2024-11-20T20:35:57,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741911_1089 (size=5523) 2024-11-20T20:35:57,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741911_1089 (size=5523) 2024-11-20T20:35:57,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741911_1089 (size=5523) 2024-11-20T20:35:57,461 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/.tmp/a/ca33b1cde15543929ef6098e4da6fb65 2024-11-20T20:35:57,485 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/.tmp/b/e8924d6125ae4089903140654290fbf1 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732134957347/Put/seqid=0 2024-11-20T20:35:57,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741912_1090 (size=5524) 2024-11-20T20:35:57,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741912_1090 (size=5524) 2024-11-20T20:35:57,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741912_1090 (size=5524) 2024-11-20T20:35:57,496 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/.tmp/b/e8924d6125ae4089903140654290fbf1 2024-11-20T20:35:57,516 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/.tmp/c/6a3649d0158a4c398ac67498967b0453 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732134957349/Put/seqid=0 2024-11-20T20:35:57,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741913_1091 (size=5457) 2024-11-20T20:35:57,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741913_1091 (size=5457) 2024-11-20T20:35:57,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741913_1091 (size=5457) 2024-11-20T20:35:57,525 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/.tmp/c/6a3649d0158a4c398ac67498967b0453 2024-11-20T20:35:57,533 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/.tmp/a/ca33b1cde15543929ef6098e4da6fb65 as hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/a/ca33b1cde15543929ef6098e4da6fb65 2024-11-20T20:35:57,539 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/a/ca33b1cde15543929ef6098e4da6fb65, entries=7, sequenceid=26, filesize=5.4 K 2024-11-20T20:35:57,540 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/.tmp/b/e8924d6125ae4089903140654290fbf1 as hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/b/e8924d6125ae4089903140654290fbf1 2024-11-20T20:35:57,547 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/b/e8924d6125ae4089903140654290fbf1, entries=7, sequenceid=26, filesize=5.4 K 2024-11-20T20:35:57,548 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/.tmp/c/6a3649d0158a4c398ac67498967b0453 as hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/c/6a3649d0158a4c398ac67498967b0453 2024-11-20T20:35:57,555 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/c/6a3649d0158a4c398ac67498967b0453, entries=6, sequenceid=26, filesize=5.3 K 2024-11-20T20:35:57,556 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for e188d891e51b57da9db6493440046b4b in 121ms, sequenceid=26, compaction requested=false; wal=null 2024-11-20T20:35:57,557 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/recovered.edits/0000000000000000026 2024-11-20T20:35:57,558 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:57,559 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:57,559 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T20:35:57,561 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:57,563 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/hbase/data/default/testReplayEditsAfterAbortingFlush/e188d891e51b57da9db6493440046b4b/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-20T20:35:57,564 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e188d891e51b57da9db6493440046b4b; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70840308, jitterRate=0.055602848529815674}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T20:35:57,565 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e188d891e51b57da9db6493440046b4b 2024-11-20T20:35:57,565 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e188d891e51b57da9db6493440046b4b: Running coprocessor pre-open hook at 1732134957423Writing region info on filesystem at 1732134957423Initializing all the Stores at 1732134957424 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134957424Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134957425 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134957425Obtaining lock to block concurrent updates at 1732134957435 (+10 ms)Preparing flush snapshotting stores in e188d891e51b57da9db6493440046b4b at 1732134957435Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732134947963.e188d891e51b57da9db6493440046b4b., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732134957435Flushing stores of testReplayEditsAfterAbortingFlush,,1732134947963.e188d891e51b57da9db6493440046b4b. at 1732134957435Flushing e188d891e51b57da9db6493440046b4b/a: creating writer at 1732134957435Flushing e188d891e51b57da9db6493440046b4b/a: appending metadata at 1732134957449 (+14 ms)Flushing e188d891e51b57da9db6493440046b4b/a: closing flushed file at 1732134957449Flushing e188d891e51b57da9db6493440046b4b/b: creating writer at 1732134957467 (+18 ms)Flushing e188d891e51b57da9db6493440046b4b/b: appending metadata at 1732134957484 (+17 ms)Flushing e188d891e51b57da9db6493440046b4b/b: closing flushed file at 1732134957484Flushing e188d891e51b57da9db6493440046b4b/c: creating writer at 1732134957502 (+18 ms)Flushing e188d891e51b57da9db6493440046b4b/c: appending metadata at 1732134957515 (+13 ms)Flushing e188d891e51b57da9db6493440046b4b/c: closing flushed file at 1732134957515Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fd25bcb: reopening flushed file at 1732134957532 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e88a437: reopening flushed file at 1732134957539 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a44303c: reopening flushed file at 1732134957547 (+8 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for e188d891e51b57da9db6493440046b4b in 121ms, sequenceid=26, compaction requested=false; wal=null at 1732134957556 (+9 ms)Cleaning up temporary data from old regions at 1732134957559 (+3 ms)Running coprocessor post-open hooks at 1732134957565 (+6 ms)Region opened successfully at 1732134957565 2024-11-20T20:35:57,597 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=417 (was 414) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:54742 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:58334 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1180650432-172.17.0.2-1732134912528:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:42804 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1180650432-172.17.0.2-1732134912528:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:54690 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:42778 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1180650432-172.17.0.2-1732134912528:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:58308 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1186 (was 1116) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=912 (was 933), ProcessCount=11 (was 11), AvailableMemoryMB=1517 (was 1559) 2024-11-20T20:35:57,598 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1186 is superior to 1024 2024-11-20T20:35:57,609 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=417, OpenFileDescriptor=1186, MaxFileDescriptor=1048576, SystemLoadAverage=912, ProcessCount=11, AvailableMemoryMB=1515 2024-11-20T20:35:57,609 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1186 is superior to 1024 2024-11-20T20:35:57,624 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:57,626 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:57,627 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T20:35:57,630 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-51297876, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/hregion-51297876, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:57,642 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-51297876/hregion-51297876.1732134957630, exclude list is [], retry=0 2024-11-20T20:35:57,645 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:57,645 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:57,645 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:57,661 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-51297876/hregion-51297876.1732134957630 2024-11-20T20:35:57,664 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39759:39759),(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:38233:38233)] 2024-11-20T20:35:57,664 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 34a9cf9fb49bf06af2bd8b5725fa6bf3, NAME => 'testDatalossWhenInputError,,1732134957625.34a9cf9fb49bf06af2bd8b5725fa6bf3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33605/hbase 2024-11-20T20:35:57,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741915_1093 (size=61) 2024-11-20T20:35:57,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741915_1093 (size=61) 2024-11-20T20:35:57,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741915_1093 (size=61) 2024-11-20T20:35:57,688 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732134957625.34a9cf9fb49bf06af2bd8b5725fa6bf3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:57,692 INFO [StoreOpener-34a9cf9fb49bf06af2bd8b5725fa6bf3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:57,696 INFO [StoreOpener-34a9cf9fb49bf06af2bd8b5725fa6bf3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34a9cf9fb49bf06af2bd8b5725fa6bf3 columnFamilyName a 2024-11-20T20:35:57,696 DEBUG [StoreOpener-34a9cf9fb49bf06af2bd8b5725fa6bf3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:57,700 INFO [StoreOpener-34a9cf9fb49bf06af2bd8b5725fa6bf3-1 {}] regionserver.HStore(327): Store=34a9cf9fb49bf06af2bd8b5725fa6bf3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:57,700 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:57,705 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:57,705 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:57,706 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:57,706 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:57,709 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:57,721 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T20:35:57,722 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 34a9cf9fb49bf06af2bd8b5725fa6bf3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67190787, jitterRate=0.0012207478284835815}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T20:35:57,723 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 34a9cf9fb49bf06af2bd8b5725fa6bf3: Writing region info on filesystem at 1732134957688Initializing all the Stores at 1732134957689 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134957689Cleaning up temporary data from old regions at 1732134957707 (+18 ms)Region opened successfully at 1732134957723 (+16 ms) 2024-11-20T20:35:57,723 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 34a9cf9fb49bf06af2bd8b5725fa6bf3, disabling compactions & flushes 2024-11-20T20:35:57,723 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732134957625.34a9cf9fb49bf06af2bd8b5725fa6bf3. 2024-11-20T20:35:57,723 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732134957625.34a9cf9fb49bf06af2bd8b5725fa6bf3. 2024-11-20T20:35:57,723 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732134957625.34a9cf9fb49bf06af2bd8b5725fa6bf3. after waiting 0 ms 2024-11-20T20:35:57,723 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732134957625.34a9cf9fb49bf06af2bd8b5725fa6bf3. 2024-11-20T20:35:57,724 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732134957625.34a9cf9fb49bf06af2bd8b5725fa6bf3. 2024-11-20T20:35:57,724 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 34a9cf9fb49bf06af2bd8b5725fa6bf3: Waiting for close lock at 1732134957723Disabling compacts and flushes for region at 1732134957723Disabling writes for close at 1732134957723Writing region close event to WAL at 1732134957724 (+1 ms)Closed at 1732134957724 2024-11-20T20:35:57,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741914_1092 (size=93) 2024-11-20T20:35:57,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741914_1092 (size=93) 2024-11-20T20:35:57,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741914_1092 (size=93) 2024-11-20T20:35:57,730 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T20:35:57,730 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-51297876:(num 1732134957630) 2024-11-20T20:35:57,730 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T20:35:57,732 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/testdatalosswheninputerror-manual,16010,1732134957623, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:57,745 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732134957623/wal.1732134957733, exclude list is [], retry=0 2024-11-20T20:35:57,748 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:57,749 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:57,749 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:57,757 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732134957623/wal.1732134957733 2024-11-20T20:35:57,760 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38233:38233),(127.0.0.1/127.0.0.1:39759:39759),(127.0.0.1/127.0.0.1:46511:46511)] 2024-11-20T20:35:57,760 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 34a9cf9fb49bf06af2bd8b5725fa6bf3, NAME => 'testDatalossWhenInputError,,1732134957625.34a9cf9fb49bf06af2bd8b5725fa6bf3.', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:35:57,761 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732134957625.34a9cf9fb49bf06af2bd8b5725fa6bf3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:57,761 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:57,761 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:57,764 INFO [StoreOpener-34a9cf9fb49bf06af2bd8b5725fa6bf3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:57,766 INFO [StoreOpener-34a9cf9fb49bf06af2bd8b5725fa6bf3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34a9cf9fb49bf06af2bd8b5725fa6bf3 columnFamilyName a 2024-11-20T20:35:57,766 DEBUG [StoreOpener-34a9cf9fb49bf06af2bd8b5725fa6bf3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:57,767 INFO [StoreOpener-34a9cf9fb49bf06af2bd8b5725fa6bf3-1 {}] regionserver.HStore(327): Store=34a9cf9fb49bf06af2bd8b5725fa6bf3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:57,768 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:57,769 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:57,770 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:57,771 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:57,771 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:57,773 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:57,774 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 34a9cf9fb49bf06af2bd8b5725fa6bf3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71638566, jitterRate=0.06749781966209412}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T20:35:57,774 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 34a9cf9fb49bf06af2bd8b5725fa6bf3: Writing region info on filesystem at 1732134957761Initializing all the Stores at 1732134957762 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134957762Cleaning up temporary data from old regions at 1732134957771 (+9 ms)Region opened successfully at 1732134957774 (+3 ms) 2024-11-20T20:35:57,785 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 34a9cf9fb49bf06af2bd8b5725fa6bf3, disabling compactions & flushes 2024-11-20T20:35:57,785 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732134957625.34a9cf9fb49bf06af2bd8b5725fa6bf3. 2024-11-20T20:35:57,785 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732134957625.34a9cf9fb49bf06af2bd8b5725fa6bf3. 2024-11-20T20:35:57,785 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732134957625.34a9cf9fb49bf06af2bd8b5725fa6bf3. after waiting 0 ms 2024-11-20T20:35:57,785 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732134957625.34a9cf9fb49bf06af2bd8b5725fa6bf3. 2024-11-20T20:35:57,792 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732134957625.34a9cf9fb49bf06af2bd8b5725fa6bf3. 2024-11-20T20:35:57,792 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732134957625.34a9cf9fb49bf06af2bd8b5725fa6bf3. 2024-11-20T20:35:57,793 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 34a9cf9fb49bf06af2bd8b5725fa6bf3: Waiting for close lock at 1732134957784Disabling compacts and flushes for region at 1732134957784Disabling writes for close at 1732134957785 (+1 ms)Writing region close event to WAL at 1732134957792 (+7 ms)Closed at 1732134957792 2024-11-20T20:35:57,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741916_1094 (size=838) 2024-11-20T20:35:57,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741916_1094 (size=838) 2024-11-20T20:35:57,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741916_1094 (size=838) 2024-11-20T20:35:57,833 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33605/hbase/WALs/testdatalosswheninputerror-manual,16010,1732134957623/wal.1732134957733, size=838 (838bytes) 2024-11-20T20:35:57,833 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33605/hbase/WALs/testdatalosswheninputerror-manual,16010,1732134957623/wal.1732134957733 2024-11-20T20:35:57,834 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33605/hbase/WALs/testdatalosswheninputerror-manual,16010,1732134957623/wal.1732134957733 after 1ms 2024-11-20T20:35:57,837 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/WALs/testdatalosswheninputerror-manual,16010,1732134957623/wal.1732134957733: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:57,837 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33605/hbase/WALs/testdatalosswheninputerror-manual,16010,1732134957623/wal.1732134957733 took 4ms 2024-11-20T20:35:57,842 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33605/hbase/WALs/testdatalosswheninputerror-manual,16010,1732134957623/wal.1732134957733 so closing down 2024-11-20T20:35:57,842 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T20:35:57,843 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732134957733.temp 2024-11-20T20:35:57,844 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3/recovered.edits/0000000000000000003-wal.1732134957733.temp 2024-11-20T20:35:57,844 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T20:35:57,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741917_1095 (size=838) 2024-11-20T20:35:57,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741917_1095 (size=838) 2024-11-20T20:35:57,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741917_1095 (size=838) 2024-11-20T20:35:57,854 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3/recovered.edits/0000000000000000003-wal.1732134957733.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-20T20:35:57,855 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3/recovered.edits/0000000000000000003-wal.1732134957733.temp to hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3/recovered.edits/0000000000000000012 2024-11-20T20:35:57,855 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:33605/hbase/WALs/testdatalosswheninputerror-manual,16010,1732134957623/wal.1732134957733, size=838, length=838, corrupted=false, cancelled=false 2024-11-20T20:35:57,855 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33605/hbase/WALs/testdatalosswheninputerror-manual,16010,1732134957623/wal.1732134957733, journal: Splitting hdfs://localhost:33605/hbase/WALs/testdatalosswheninputerror-manual,16010,1732134957623/wal.1732134957733, size=838 (838bytes) at 1732134957833Finishing writing output for hdfs://localhost:33605/hbase/WALs/testdatalosswheninputerror-manual,16010,1732134957623/wal.1732134957733 so closing down at 1732134957842 (+9 ms)Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3/recovered.edits/0000000000000000003-wal.1732134957733.temp at 1732134957844 (+2 ms)3 split writer threads finished at 1732134957844Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3/recovered.edits/0000000000000000003-wal.1732134957733.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732134957854 (+10 ms)Rename recovered edits hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3/recovered.edits/0000000000000000003-wal.1732134957733.temp to hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3/recovered.edits/0000000000000000012 at 1732134957855 (+1 ms)Processed 10 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:33605/hbase/WALs/testdatalosswheninputerror-manual,16010,1732134957623/wal.1732134957733, size=838, length=838, corrupted=false, cancelled=false at 1732134957855 2024-11-20T20:35:57,857 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33605/hbase/WALs/testdatalosswheninputerror-manual,16010,1732134957623/wal.1732134957733 to hdfs://localhost:33605/hbase/oldWALs/wal.1732134957733 2024-11-20T20:35:57,858 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3/recovered.edits/0000000000000000012 2024-11-20T20:35:57,862 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:58,360 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T20:35:58,362 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/testdatalosswheninputerror-manual,16010,1732134957623, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:58,381 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732134957623/wal.1732134958363, exclude list is [], retry=0 2024-11-20T20:35:58,383 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:58,384 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:58,384 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:58,386 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732134957623/wal.1732134958363 2024-11-20T20:35:58,386 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39759:39759),(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:38233:38233)] 2024-11-20T20:35:58,386 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 34a9cf9fb49bf06af2bd8b5725fa6bf3, NAME => 'testDatalossWhenInputError,,1732134957625.34a9cf9fb49bf06af2bd8b5725fa6bf3.', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:35:58,386 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732134957625.34a9cf9fb49bf06af2bd8b5725fa6bf3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:58,386 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:58,386 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:58,390 INFO [StoreOpener-34a9cf9fb49bf06af2bd8b5725fa6bf3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:58,391 INFO [StoreOpener-34a9cf9fb49bf06af2bd8b5725fa6bf3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34a9cf9fb49bf06af2bd8b5725fa6bf3 columnFamilyName a 2024-11-20T20:35:58,392 DEBUG [StoreOpener-34a9cf9fb49bf06af2bd8b5725fa6bf3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:58,393 INFO [StoreOpener-34a9cf9fb49bf06af2bd8b5725fa6bf3-1 {}] regionserver.HStore(327): Store=34a9cf9fb49bf06af2bd8b5725fa6bf3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:58,393 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:58,394 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:58,395 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:58,396 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3/recovered.edits/0000000000000000012 2024-11-20T20:35:58,398 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:35:58,399 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3/recovered.edits/0000000000000000012 2024-11-20T20:35:58,400 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 34a9cf9fb49bf06af2bd8b5725fa6bf3 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-20T20:35:58,419 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3/.tmp/a/9aa5d0aae3ab4807a3eb6276ce220066 is 79, key is testDatalossWhenInputError/a:x0/1732134957775/Put/seqid=0 2024-11-20T20:35:58,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741919_1097 (size=5808) 2024-11-20T20:35:58,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741919_1097 (size=5808) 2024-11-20T20:35:58,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741919_1097 (size=5808) 2024-11-20T20:35:58,827 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3/.tmp/a/9aa5d0aae3ab4807a3eb6276ce220066 2024-11-20T20:35:58,842 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3/.tmp/a/9aa5d0aae3ab4807a3eb6276ce220066 as hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3/a/9aa5d0aae3ab4807a3eb6276ce220066 2024-11-20T20:35:58,850 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3/a/9aa5d0aae3ab4807a3eb6276ce220066, entries=10, sequenceid=12, filesize=5.7 K 2024-11-20T20:35:58,850 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 34a9cf9fb49bf06af2bd8b5725fa6bf3 in 451ms, sequenceid=12, compaction requested=false; wal=null 2024-11-20T20:35:58,851 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3/recovered.edits/0000000000000000012 2024-11-20T20:35:58,852 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:58,852 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:58,854 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:58,856 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-20T20:35:58,857 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 34a9cf9fb49bf06af2bd8b5725fa6bf3; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58856874, jitterRate=-0.12296423316001892}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T20:35:58,858 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 34a9cf9fb49bf06af2bd8b5725fa6bf3: Writing region info on filesystem at 1732134958386Initializing all the Stores at 1732134958389 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134958389Obtaining lock to block concurrent updates at 1732134958400 (+11 ms)Preparing flush snapshotting stores in 34a9cf9fb49bf06af2bd8b5725fa6bf3 at 1732134958400Finished memstore snapshotting testDatalossWhenInputError,,1732134957625.34a9cf9fb49bf06af2bd8b5725fa6bf3., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732134958400Flushing stores of testDatalossWhenInputError,,1732134957625.34a9cf9fb49bf06af2bd8b5725fa6bf3. at 1732134958400Flushing 34a9cf9fb49bf06af2bd8b5725fa6bf3/a: creating writer at 1732134958400Flushing 34a9cf9fb49bf06af2bd8b5725fa6bf3/a: appending metadata at 1732134958419 (+19 ms)Flushing 34a9cf9fb49bf06af2bd8b5725fa6bf3/a: closing flushed file at 1732134958419Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@559d0560: reopening flushed file at 1732134958840 (+421 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 34a9cf9fb49bf06af2bd8b5725fa6bf3 in 451ms, sequenceid=12, compaction requested=false; wal=null at 1732134958850 (+10 ms)Cleaning up temporary data from old regions at 1732134958852 (+2 ms)Region opened successfully at 1732134958858 (+6 ms) 2024-11-20T20:35:58,861 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 34a9cf9fb49bf06af2bd8b5725fa6bf3, NAME => 'testDatalossWhenInputError,,1732134957625.34a9cf9fb49bf06af2bd8b5725fa6bf3.', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:35:58,861 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732134957625.34a9cf9fb49bf06af2bd8b5725fa6bf3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:58,861 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:58,861 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:58,862 INFO [StoreOpener-34a9cf9fb49bf06af2bd8b5725fa6bf3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:58,863 INFO [StoreOpener-34a9cf9fb49bf06af2bd8b5725fa6bf3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34a9cf9fb49bf06af2bd8b5725fa6bf3 columnFamilyName a 2024-11-20T20:35:58,863 DEBUG [StoreOpener-34a9cf9fb49bf06af2bd8b5725fa6bf3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:58,868 DEBUG [StoreOpener-34a9cf9fb49bf06af2bd8b5725fa6bf3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3/a/9aa5d0aae3ab4807a3eb6276ce220066 2024-11-20T20:35:58,868 INFO [StoreOpener-34a9cf9fb49bf06af2bd8b5725fa6bf3-1 {}] regionserver.HStore(327): Store=34a9cf9fb49bf06af2bd8b5725fa6bf3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:58,868 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:58,869 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:58,870 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:58,870 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:58,870 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:58,872 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 34a9cf9fb49bf06af2bd8b5725fa6bf3 2024-11-20T20:35:58,874 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/hbase/data/default/testDatalossWhenInputError/34a9cf9fb49bf06af2bd8b5725fa6bf3/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-20T20:35:58,875 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 34a9cf9fb49bf06af2bd8b5725fa6bf3; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59534314, jitterRate=-0.11286959052085876}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T20:35:58,875 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 34a9cf9fb49bf06af2bd8b5725fa6bf3: Writing region info on filesystem at 1732134958861Initializing all the Stores at 1732134958862 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134958862Cleaning up temporary data from old regions at 1732134958870 (+8 ms)Region opened successfully at 1732134958875 (+5 ms) 2024-11-20T20:35:58,893 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=427 (was 417) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1180650432-172.17.0.2-1732134912528:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:54742 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:58334 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1180650432-172.17.0.2-1732134912528:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:58406 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:42884 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:42778 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:54804 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1180650432-172.17.0.2-1732134912528:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1268 (was 1186) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=912 (was 912), ProcessCount=11 (was 11), AvailableMemoryMB=1435 (was 1515) 2024-11-20T20:35:58,893 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1268 is superior to 1024 2024-11-20T20:35:58,908 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=427, OpenFileDescriptor=1268, MaxFileDescriptor=1048576, SystemLoadAverage=912, ProcessCount=11, AvailableMemoryMB=1433 2024-11-20T20:35:58,908 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1268 is superior to 1024 2024-11-20T20:35:58,923 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:58,926 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:35:58,926 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T20:35:58,929 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-66614615, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/hregion-66614615, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:58,943 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-66614615/hregion-66614615.1732134958929, exclude list is [], retry=0 2024-11-20T20:35:58,946 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:58,946 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:58,947 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:58,948 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-66614615/hregion-66614615.1732134958929 2024-11-20T20:35:58,948 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39759:39759),(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:38233:38233)] 2024-11-20T20:35:58,949 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 93e03e0c1bb581f06d9dba15a3a69aaa, NAME => 'testCompactedBulkLoadedFiles,,1732134958924.93e03e0c1bb581f06d9dba15a3a69aaa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33605/hbase 2024-11-20T20:35:58,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741921_1099 (size=63) 2024-11-20T20:35:58,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741921_1099 (size=63) 2024-11-20T20:35:58,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741921_1099 (size=63) 2024-11-20T20:35:58,958 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732134958924.93e03e0c1bb581f06d9dba15a3a69aaa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:58,960 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:35:58,962 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93e03e0c1bb581f06d9dba15a3a69aaa columnFamilyName a 2024-11-20T20:35:58,962 DEBUG [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:58,962 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.HStore(327): Store=93e03e0c1bb581f06d9dba15a3a69aaa/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:58,962 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:35:58,964 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93e03e0c1bb581f06d9dba15a3a69aaa columnFamilyName b 2024-11-20T20:35:58,964 DEBUG [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:58,964 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.HStore(327): Store=93e03e0c1bb581f06d9dba15a3a69aaa/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:58,964 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:35:58,966 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93e03e0c1bb581f06d9dba15a3a69aaa columnFamilyName c 2024-11-20T20:35:58,966 DEBUG [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:58,966 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.HStore(327): Store=93e03e0c1bb581f06d9dba15a3a69aaa/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:58,966 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:35:58,967 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:35:58,967 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:35:58,968 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:35:58,968 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:35:58,969 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T20:35:58,970 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:35:58,972 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T20:35:58,972 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 93e03e0c1bb581f06d9dba15a3a69aaa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62792899, jitterRate=-0.0643128901720047}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T20:35:58,972 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 93e03e0c1bb581f06d9dba15a3a69aaa: Writing region info on filesystem at 1732134958958Initializing all the Stores at 1732134958959 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134958959Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134958960 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134958960Cleaning up temporary data from old regions at 1732134958968 (+8 ms)Region opened successfully at 1732134958972 (+4 ms) 2024-11-20T20:35:58,973 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 93e03e0c1bb581f06d9dba15a3a69aaa, disabling compactions & flushes 2024-11-20T20:35:58,973 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732134958924.93e03e0c1bb581f06d9dba15a3a69aaa. 2024-11-20T20:35:58,973 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732134958924.93e03e0c1bb581f06d9dba15a3a69aaa. 2024-11-20T20:35:58,973 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732134958924.93e03e0c1bb581f06d9dba15a3a69aaa. after waiting 0 ms 2024-11-20T20:35:58,973 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732134958924.93e03e0c1bb581f06d9dba15a3a69aaa. 2024-11-20T20:35:58,973 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732134958924.93e03e0c1bb581f06d9dba15a3a69aaa. 2024-11-20T20:35:58,973 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 93e03e0c1bb581f06d9dba15a3a69aaa: Waiting for close lock at 1732134958973Disabling compacts and flushes for region at 1732134958973Disabling writes for close at 1732134958973Writing region close event to WAL at 1732134958973Closed at 1732134958973 2024-11-20T20:35:58,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741920_1098 (size=93) 2024-11-20T20:35:58,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741920_1098 (size=93) 2024-11-20T20:35:58,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741920_1098 (size=93) 2024-11-20T20:35:58,977 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T20:35:58,977 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-66614615:(num 1732134958929) 2024-11-20T20:35:58,977 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T20:35:58,979 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732134958922, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:35:58,994 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732134958922/wal.1732134958979, exclude list is [], retry=0 2024-11-20T20:35:58,997 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:35:58,997 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:35:58,998 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:35:59,002 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732134958922/wal.1732134958979 2024-11-20T20:35:59,003 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:38233:38233),(127.0.0.1/127.0.0.1:39759:39759)] 2024-11-20T20:35:59,003 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 93e03e0c1bb581f06d9dba15a3a69aaa, NAME => 'testCompactedBulkLoadedFiles,,1732134958924.93e03e0c1bb581f06d9dba15a3a69aaa.', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:35:59,003 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732134958924.93e03e0c1bb581f06d9dba15a3a69aaa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:35:59,003 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:35:59,003 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:35:59,005 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:35:59,006 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93e03e0c1bb581f06d9dba15a3a69aaa columnFamilyName a 2024-11-20T20:35:59,006 DEBUG [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:59,009 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.HStore(327): Store=93e03e0c1bb581f06d9dba15a3a69aaa/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:59,009 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:35:59,011 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93e03e0c1bb581f06d9dba15a3a69aaa columnFamilyName b 2024-11-20T20:35:59,012 DEBUG [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:59,012 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.HStore(327): Store=93e03e0c1bb581f06d9dba15a3a69aaa/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:59,012 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:35:59,013 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93e03e0c1bb581f06d9dba15a3a69aaa columnFamilyName c 2024-11-20T20:35:59,013 DEBUG [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:35:59,014 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.HStore(327): Store=93e03e0c1bb581f06d9dba15a3a69aaa/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:35:59,014 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:35:59,015 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:35:59,016 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:35:59,017 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:35:59,017 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:35:59,018 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T20:35:59,019 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:35:59,020 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 93e03e0c1bb581f06d9dba15a3a69aaa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67895595, jitterRate=0.01172320544719696}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T20:35:59,021 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 93e03e0c1bb581f06d9dba15a3a69aaa: Writing region info on filesystem at 1732134959003Initializing all the Stores at 1732134959005 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134959005Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134959005Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134959005Cleaning up temporary data from old regions at 1732134959017 (+12 ms)Region opened successfully at 1732134959021 (+4 ms) 2024-11-20T20:35:59,025 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732134959025/Put/seqid=0 2024-11-20T20:35:59,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741923_1101 (size=4875) 2024-11-20T20:35:59,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741923_1101 (size=4875) 2024-11-20T20:35:59,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741923_1101 (size=4875) 2024-11-20T20:35:59,033 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732134959033/Put/seqid=0 2024-11-20T20:35:59,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741924_1102 (size=4875) 2024-11-20T20:35:59,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741924_1102 (size=4875) 2024-11-20T20:35:59,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741924_1102 (size=4875) 2024-11-20T20:35:59,041 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732134959041/Put/seqid=0 2024-11-20T20:35:59,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741925_1103 (size=4875) 2024-11-20T20:35:59,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741925_1103 (size=4875) 2024-11-20T20:35:59,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741925_1103 (size=4875) 2024-11-20T20:35:59,048 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33605/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 93e03e0c1bb581f06d9dba15a3a69aaa/a 2024-11-20T20:35:59,051 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-20T20:35:59,051 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-20T20:35:59,051 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33605/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 93e03e0c1bb581f06d9dba15a3a69aaa/a 2024-11-20T20:35:59,054 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-20T20:35:59,054 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-20T20:35:59,055 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33605/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 93e03e0c1bb581f06d9dba15a3a69aaa/a 2024-11-20T20:35:59,058 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-20T20:35:59,058 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-20T20:35:59,058 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 93e03e0c1bb581f06d9dba15a3a69aaa 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-20T20:35:59,076 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/.tmp/a/3049b6964be54be9bdd92017213c1cc5 is 55, key is testCompactedBulkLoadedFiles/a:a/1732134959021/Put/seqid=0 2024-11-20T20:35:59,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741926_1104 (size=5107) 2024-11-20T20:35:59,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741926_1104 (size=5107) 2024-11-20T20:35:59,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741926_1104 (size=5107) 2024-11-20T20:35:59,083 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/.tmp/a/3049b6964be54be9bdd92017213c1cc5 2024-11-20T20:35:59,090 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/.tmp/a/3049b6964be54be9bdd92017213c1cc5 as hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/3049b6964be54be9bdd92017213c1cc5 2024-11-20T20:35:59,095 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/3049b6964be54be9bdd92017213c1cc5, entries=1, sequenceid=4, filesize=5.0 K 2024-11-20T20:35:59,097 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 93e03e0c1bb581f06d9dba15a3a69aaa in 39ms, sequenceid=4, compaction requested=false 2024-11-20T20:35:59,097 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 93e03e0c1bb581f06d9dba15a3a69aaa: 2024-11-20T20:35:59,098 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/d064c8603c7a4efc9cc8b071cf9ec10d_SeqId_4_ 2024-11-20T20:35:59,099 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/f8d17fef7c944dc098049ecb793d04ae_SeqId_4_ 2024-11-20T20:35:59,100 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/9eb374cd83ee4df19fa51884730a5938_SeqId_4_ 2024-11-20T20:35:59,101 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33605/hbase/testCompactedBulkLoadedFiles/hfile0 into 93e03e0c1bb581f06d9dba15a3a69aaa/a as hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/d064c8603c7a4efc9cc8b071cf9ec10d_SeqId_4_ - updating store file list. 2024-11-20T20:35:59,107 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for d064c8603c7a4efc9cc8b071cf9ec10d_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T20:35:59,107 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/d064c8603c7a4efc9cc8b071cf9ec10d_SeqId_4_ into 93e03e0c1bb581f06d9dba15a3a69aaa/a 2024-11-20T20:35:59,108 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33605/hbase/testCompactedBulkLoadedFiles/hfile0 into 93e03e0c1bb581f06d9dba15a3a69aaa/a (new location: hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/d064c8603c7a4efc9cc8b071cf9ec10d_SeqId_4_) 2024-11-20T20:35:59,109 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33605/hbase/testCompactedBulkLoadedFiles/hfile1 into 93e03e0c1bb581f06d9dba15a3a69aaa/a as hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/f8d17fef7c944dc098049ecb793d04ae_SeqId_4_ - updating store file list. 2024-11-20T20:35:59,114 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for f8d17fef7c944dc098049ecb793d04ae_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T20:35:59,115 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/f8d17fef7c944dc098049ecb793d04ae_SeqId_4_ into 93e03e0c1bb581f06d9dba15a3a69aaa/a 2024-11-20T20:35:59,115 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33605/hbase/testCompactedBulkLoadedFiles/hfile1 into 93e03e0c1bb581f06d9dba15a3a69aaa/a (new location: hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/f8d17fef7c944dc098049ecb793d04ae_SeqId_4_) 2024-11-20T20:35:59,116 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33605/hbase/testCompactedBulkLoadedFiles/hfile2 into 93e03e0c1bb581f06d9dba15a3a69aaa/a as hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/9eb374cd83ee4df19fa51884730a5938_SeqId_4_ - updating store file list. 2024-11-20T20:35:59,121 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 9eb374cd83ee4df19fa51884730a5938_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T20:35:59,121 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/9eb374cd83ee4df19fa51884730a5938_SeqId_4_ into 93e03e0c1bb581f06d9dba15a3a69aaa/a 2024-11-20T20:35:59,121 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33605/hbase/testCompactedBulkLoadedFiles/hfile2 into 93e03e0c1bb581f06d9dba15a3a69aaa/a (new location: hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/9eb374cd83ee4df19fa51884730a5938_SeqId_4_) 2024-11-20T20:35:59,130 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T20:35:59,130 DEBUG [Time-limited test {}] regionserver.HStore(1541): 93e03e0c1bb581f06d9dba15a3a69aaa/a is initiating major compaction (all files) 2024-11-20T20:35:59,131 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 93e03e0c1bb581f06d9dba15a3a69aaa/a in testCompactedBulkLoadedFiles,,1732134958924.93e03e0c1bb581f06d9dba15a3a69aaa. 2024-11-20T20:35:59,131 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/3049b6964be54be9bdd92017213c1cc5, hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/d064c8603c7a4efc9cc8b071cf9ec10d_SeqId_4_, hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/f8d17fef7c944dc098049ecb793d04ae_SeqId_4_, hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/9eb374cd83ee4df19fa51884730a5938_SeqId_4_] into tmpdir=hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/.tmp, totalSize=19.3 K 2024-11-20T20:35:59,131 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3049b6964be54be9bdd92017213c1cc5, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732134959021 2024-11-20T20:35:59,132 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d064c8603c7a4efc9cc8b071cf9ec10d_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-20T20:35:59,132 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting f8d17fef7c944dc098049ecb793d04ae_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-20T20:35:59,133 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 9eb374cd83ee4df19fa51884730a5938_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-20T20:35:59,156 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/.tmp/a/87086dccb7a342f984e5981b63fd0325 is 55, key is testCompactedBulkLoadedFiles/a:a/1732134959021/Put/seqid=0 2024-11-20T20:35:59,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741927_1105 (size=6154) 2024-11-20T20:35:59,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741927_1105 (size=6154) 2024-11-20T20:35:59,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741927_1105 (size=6154) 2024-11-20T20:35:59,178 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/.tmp/a/87086dccb7a342f984e5981b63fd0325 as hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/87086dccb7a342f984e5981b63fd0325 2024-11-20T20:35:59,185 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 93e03e0c1bb581f06d9dba15a3a69aaa/a of 93e03e0c1bb581f06d9dba15a3a69aaa into 87086dccb7a342f984e5981b63fd0325(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T20:35:59,185 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 93e03e0c1bb581f06d9dba15a3a69aaa: 2024-11-20T20:35:59,185 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-20T20:35:59,185 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-20T20:35:59,229 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33605/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732134958922/wal.1732134958979, size=0 (0bytes) 2024-11-20T20:35:59,229 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33605/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732134958922/wal.1732134958979 might be still open, length is 0 2024-11-20T20:35:59,229 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33605/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732134958922/wal.1732134958979 2024-11-20T20:35:59,230 WARN [IPC Server handler 2 on default port 33605 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732134958922/wal.1732134958979 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-20T20:35:59,230 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33605/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732134958922/wal.1732134958979 after 1ms 2024-11-20T20:36:02,015 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:42924 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:44519:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42924 dst: /127.0.0.1:44519 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:44519 remote=/127.0.0.1:42924]. Total timeout mills is 60000, 57171 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:36:02,016 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:54858 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:35033:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54858 dst: /127.0.0.1:35033 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:36:02,015 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:58430 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:40323:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58430 dst: /127.0.0.1:40323 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:36:02,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741922_1106 (size=1368) 2024-11-20T20:36:02,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741922_1106 (size=1368) 2024-11-20T20:36:02,864 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T20:36:03,231 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33605/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732134958922/wal.1732134958979 after 4002ms 2024-11-20T20:36:03,235 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732134958922/wal.1732134958979: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:36:03,235 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33605/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732134958922/wal.1732134958979 took 4006ms 2024-11-20T20:36:03,238 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33605/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732134958922/wal.1732134958979; continuing. 2024-11-20T20:36:03,238 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33605/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732134958922/wal.1732134958979 so closing down 2024-11-20T20:36:03,238 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T20:36:03,240 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732134958979.temp 2024-11-20T20:36:03,242 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/recovered.edits/0000000000000000003-wal.1732134958979.temp 2024-11-20T20:36:03,242 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T20:36:03,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741928_1107 (size=635) 2024-11-20T20:36:03,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741928_1107 (size=635) 2024-11-20T20:36:03,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741928_1107 (size=635) 2024-11-20T20:36:03,255 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/recovered.edits/0000000000000000003-wal.1732134958979.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-20T20:36:03,256 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/recovered.edits/0000000000000000003-wal.1732134958979.temp to hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/recovered.edits/0000000000000000008 2024-11-20T20:36:03,257 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 21 ms; skipped=3; WAL=hdfs://localhost:33605/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732134958922/wal.1732134958979, size=0, length=0, corrupted=false, cancelled=false 2024-11-20T20:36:03,257 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33605/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732134958922/wal.1732134958979, journal: Splitting hdfs://localhost:33605/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732134958922/wal.1732134958979, size=0 (0bytes) at 1732134959229Finishing writing output for hdfs://localhost:33605/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732134958922/wal.1732134958979 so closing down at 1732134963238 (+4009 ms)Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/recovered.edits/0000000000000000003-wal.1732134958979.temp at 1732134963242 (+4 ms)3 split writer threads finished at 1732134963242Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/recovered.edits/0000000000000000003-wal.1732134958979.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732134963255 (+13 ms)Rename recovered edits hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/recovered.edits/0000000000000000003-wal.1732134958979.temp to hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/recovered.edits/0000000000000000008 at 1732134963256 (+1 ms)Processed 5 edits across 1 Regions in 21 ms; skipped=3; WAL=hdfs://localhost:33605/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732134958922/wal.1732134958979, size=0, length=0, corrupted=false, cancelled=false at 1732134963257 (+1 ms) 2024-11-20T20:36:03,258 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33605/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732134958922/wal.1732134958979 to hdfs://localhost:33605/hbase/oldWALs/wal.1732134958979 2024-11-20T20:36:03,259 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/recovered.edits/0000000000000000008 2024-11-20T20:36:03,259 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T20:36:03,261 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732134958922, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:36:03,273 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732134958922/wal.1732134963261, exclude list is [], retry=0 2024-11-20T20:36:03,275 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:36:03,276 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:36:03,276 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:36:03,278 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732134958922/wal.1732134963261 2024-11-20T20:36:03,278 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38233:38233),(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:39759:39759)] 2024-11-20T20:36:03,278 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 93e03e0c1bb581f06d9dba15a3a69aaa, NAME => 'testCompactedBulkLoadedFiles,,1732134958924.93e03e0c1bb581f06d9dba15a3a69aaa.', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:36:03,278 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732134958924.93e03e0c1bb581f06d9dba15a3a69aaa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:36:03,279 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:36:03,279 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:36:03,280 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:36:03,281 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93e03e0c1bb581f06d9dba15a3a69aaa columnFamilyName a 2024-11-20T20:36:03,281 DEBUG [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:36:03,288 DEBUG [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/3049b6964be54be9bdd92017213c1cc5 2024-11-20T20:36:03,292 DEBUG [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/87086dccb7a342f984e5981b63fd0325 2024-11-20T20:36:03,297 DEBUG [StoreFileOpener-93e03e0c1bb581f06d9dba15a3a69aaa-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 9eb374cd83ee4df19fa51884730a5938_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T20:36:03,297 DEBUG [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/9eb374cd83ee4df19fa51884730a5938_SeqId_4_ 2024-11-20T20:36:03,302 DEBUG [StoreFileOpener-93e03e0c1bb581f06d9dba15a3a69aaa-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for d064c8603c7a4efc9cc8b071cf9ec10d_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T20:36:03,302 DEBUG [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/d064c8603c7a4efc9cc8b071cf9ec10d_SeqId_4_ 2024-11-20T20:36:03,307 DEBUG [StoreFileOpener-93e03e0c1bb581f06d9dba15a3a69aaa-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for f8d17fef7c944dc098049ecb793d04ae_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T20:36:03,307 DEBUG [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/f8d17fef7c944dc098049ecb793d04ae_SeqId_4_ 2024-11-20T20:36:03,307 WARN [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/3049b6964be54be9bdd92017213c1cc5 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7c184faf 2024-11-20T20:36:03,307 WARN [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/9eb374cd83ee4df19fa51884730a5938_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7c184faf 2024-11-20T20:36:03,307 WARN [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/d064c8603c7a4efc9cc8b071cf9ec10d_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7c184faf 2024-11-20T20:36:03,307 WARN [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/f8d17fef7c944dc098049ecb793d04ae_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7c184faf 2024-11-20T20:36:03,307 DEBUG [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/3049b6964be54be9bdd92017213c1cc5, hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/9eb374cd83ee4df19fa51884730a5938_SeqId_4_, hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/d064c8603c7a4efc9cc8b071cf9ec10d_SeqId_4_, hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/f8d17fef7c944dc098049ecb793d04ae_SeqId_4_] to archive 2024-11-20T20:36:03,308 DEBUG [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T20:36:03,310 DEBUG [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/3049b6964be54be9bdd92017213c1cc5 to hdfs://localhost:33605/hbase/archive/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/3049b6964be54be9bdd92017213c1cc5 2024-11-20T20:36:03,311 DEBUG [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/9eb374cd83ee4df19fa51884730a5938_SeqId_4_ to hdfs://localhost:33605/hbase/archive/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/9eb374cd83ee4df19fa51884730a5938_SeqId_4_ 2024-11-20T20:36:03,313 DEBUG [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/d064c8603c7a4efc9cc8b071cf9ec10d_SeqId_4_ to hdfs://localhost:33605/hbase/archive/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/d064c8603c7a4efc9cc8b071cf9ec10d_SeqId_4_ 2024-11-20T20:36:03,314 DEBUG [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/f8d17fef7c944dc098049ecb793d04ae_SeqId_4_ to hdfs://localhost:33605/hbase/archive/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/a/f8d17fef7c944dc098049ecb793d04ae_SeqId_4_ 2024-11-20T20:36:03,314 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.HStore(327): Store=93e03e0c1bb581f06d9dba15a3a69aaa/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:36:03,314 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:36:03,315 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93e03e0c1bb581f06d9dba15a3a69aaa columnFamilyName b 2024-11-20T20:36:03,315 DEBUG [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:36:03,316 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.HStore(327): Store=93e03e0c1bb581f06d9dba15a3a69aaa/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:36:03,316 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:36:03,317 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93e03e0c1bb581f06d9dba15a3a69aaa columnFamilyName c 2024-11-20T20:36:03,317 DEBUG [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:36:03,318 INFO [StoreOpener-93e03e0c1bb581f06d9dba15a3a69aaa-1 {}] regionserver.HStore(327): Store=93e03e0c1bb581f06d9dba15a3a69aaa/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:36:03,318 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:36:03,318 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:36:03,320 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:36:03,320 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/recovered.edits/0000000000000000008 2024-11-20T20:36:03,322 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:36:03,325 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 93e03e0c1bb581f06d9dba15a3a69aaa : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "93e03e0c1bb581f06d9dba15a3a69aaa" family_name: "a" compaction_input: "3049b6964be54be9bdd92017213c1cc5" compaction_input: "d064c8603c7a4efc9cc8b071cf9ec10d_SeqId_4_" compaction_input: "f8d17fef7c944dc098049ecb793d04ae_SeqId_4_" compaction_input: "9eb374cd83ee4df19fa51884730a5938_SeqId_4_" compaction_output: "87086dccb7a342f984e5981b63fd0325" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732134958924.93e03e0c1bb581f06d9dba15a3a69aaa." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-20T20:36:03,325 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-20T20:36:03,325 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/recovered.edits/0000000000000000008 2024-11-20T20:36:03,326 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/recovered.edits/0000000000000000008 2024-11-20T20:36:03,327 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:36:03,327 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:36:03,328 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T20:36:03,330 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 93e03e0c1bb581f06d9dba15a3a69aaa 2024-11-20T20:36:03,337 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/hbase/data/default/testCompactedBulkLoadedFiles/93e03e0c1bb581f06d9dba15a3a69aaa/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-20T20:36:03,338 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 93e03e0c1bb581f06d9dba15a3a69aaa; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66554005, jitterRate=-0.008268043398857117}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T20:36:03,338 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 93e03e0c1bb581f06d9dba15a3a69aaa: Writing region info on filesystem at 1732134963279Initializing all the Stores at 1732134963280 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134963280Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134963280Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134963280Cleaning up temporary data from old regions at 1732134963328 (+48 ms)Region opened successfully at 1732134963338 (+10 ms) 2024-11-20T20:36:03,341 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 93e03e0c1bb581f06d9dba15a3a69aaa, disabling compactions & flushes 2024-11-20T20:36:03,341 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732134958924.93e03e0c1bb581f06d9dba15a3a69aaa. 2024-11-20T20:36:03,341 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732134958924.93e03e0c1bb581f06d9dba15a3a69aaa. 2024-11-20T20:36:03,341 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732134958924.93e03e0c1bb581f06d9dba15a3a69aaa. after waiting 0 ms 2024-11-20T20:36:03,342 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732134958924.93e03e0c1bb581f06d9dba15a3a69aaa. 2024-11-20T20:36:03,342 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732134958924.93e03e0c1bb581f06d9dba15a3a69aaa. 2024-11-20T20:36:03,342 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 93e03e0c1bb581f06d9dba15a3a69aaa: Waiting for close lock at 1732134963341Disabling compacts and flushes for region at 1732134963341Disabling writes for close at 1732134963342 (+1 ms)Writing region close event to WAL at 1732134963342Closed at 1732134963342 2024-11-20T20:36:03,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741929_1108 (size=93) 2024-11-20T20:36:03,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741929_1108 (size=93) 2024-11-20T20:36:03,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741929_1108 (size=93) 2024-11-20T20:36:03,354 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T20:36:03,354 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732134963261) 2024-11-20T20:36:03,373 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=434 (was 427) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:33605 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-945141813_22 at /127.0.0.1:54698 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-945141813_22 at /127.0.0.1:50720 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-945141813_22 at /127.0.0.1:54342 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:33605 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1344 (was 1268) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=879 (was 912), ProcessCount=11 (was 11), AvailableMemoryMB=1432 (was 1433) 2024-11-20T20:36:03,374 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1344 is superior to 1024 2024-11-20T20:36:03,388 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=434, OpenFileDescriptor=1344, MaxFileDescriptor=1048576, SystemLoadAverage=879, ProcessCount=11, AvailableMemoryMB=1431 2024-11-20T20:36:03,388 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1344 is superior to 1024 2024-11-20T20:36:03,409 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:36:03,412 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T20:36:03,413 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T20:36:03,415 INFO [master/d7d59561a64e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-20T20:36:03,415 INFO [master/d7d59561a64e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-20T20:36:03,419 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-59126115, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/hregion-59126115, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:36:03,438 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-59126115/hregion-59126115.1732134963420, exclude list is [], retry=0 2024-11-20T20:36:03,441 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:36:03,442 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:36:03,444 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:36:03,448 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-59126115/hregion-59126115.1732134963420 2024-11-20T20:36:03,448 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:38233:38233),(127.0.0.1/127.0.0.1:39759:39759)] 2024-11-20T20:36:03,448 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6477a8e6578573788f81f020f6167a1a, NAME => 'testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33605/hbase 2024-11-20T20:36:03,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741931_1110 (size=67) 2024-11-20T20:36:03,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741931_1110 (size=67) 2024-11-20T20:36:03,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741931_1110 (size=67) 2024-11-20T20:36:03,472 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:36:03,480 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:03,487 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6477a8e6578573788f81f020f6167a1a columnFamilyName a 2024-11-20T20:36:03,487 DEBUG [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:36:03,490 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.HStore(327): Store=6477a8e6578573788f81f020f6167a1a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:36:03,490 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:03,494 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6477a8e6578573788f81f020f6167a1a columnFamilyName b 2024-11-20T20:36:03,494 DEBUG [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:36:03,495 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.HStore(327): Store=6477a8e6578573788f81f020f6167a1a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:36:03,495 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:03,498 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6477a8e6578573788f81f020f6167a1a columnFamilyName c 2024-11-20T20:36:03,498 DEBUG [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:36:03,499 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.HStore(327): Store=6477a8e6578573788f81f020f6167a1a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:36:03,499 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:03,500 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:03,500 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:03,501 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:03,501 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:03,502 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T20:36:03,504 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:03,512 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T20:36:03,513 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6477a8e6578573788f81f020f6167a1a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73416213, jitterRate=0.09398682415485382}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T20:36:03,513 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6477a8e6578573788f81f020f6167a1a: Writing region info on filesystem at 1732134963472Initializing all the Stores at 1732134963473 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134963473Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134963480 (+7 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134963480Cleaning up temporary data from old regions at 1732134963501 (+21 ms)Region opened successfully at 1732134963513 (+12 ms) 2024-11-20T20:36:03,513 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6477a8e6578573788f81f020f6167a1a, disabling compactions & flushes 2024-11-20T20:36:03,513 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a. 2024-11-20T20:36:03,514 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a. 2024-11-20T20:36:03,514 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a. after waiting 0 ms 2024-11-20T20:36:03,514 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a. 2024-11-20T20:36:03,514 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a. 2024-11-20T20:36:03,514 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6477a8e6578573788f81f020f6167a1a: Waiting for close lock at 1732134963513Disabling compacts and flushes for region at 1732134963513Disabling writes for close at 1732134963514 (+1 ms)Writing region close event to WAL at 1732134963514Closed at 1732134963514 2024-11-20T20:36:03,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741930_1109 (size=93) 2024-11-20T20:36:03,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741930_1109 (size=93) 2024-11-20T20:36:03,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741930_1109 (size=93) 2024-11-20T20:36:03,537 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T20:36:03,537 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-59126115:(num 1732134963420) 2024-11-20T20:36:03,537 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T20:36:03,539 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:36:03,556 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134963540, exclude list is [], retry=0 2024-11-20T20:36:03,563 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:36:03,564 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:36:03,564 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:36:03,582 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134963540 2024-11-20T20:36:03,584 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:39759:39759),(127.0.0.1/127.0.0.1:38233:38233)] 2024-11-20T20:36:03,584 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6477a8e6578573788f81f020f6167a1a, NAME => 'testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a.', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:36:03,585 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:36:03,585 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:03,585 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:03,625 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:03,626 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6477a8e6578573788f81f020f6167a1a columnFamilyName a 2024-11-20T20:36:03,626 DEBUG [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:36:03,627 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.HStore(327): Store=6477a8e6578573788f81f020f6167a1a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:36:03,627 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:03,628 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6477a8e6578573788f81f020f6167a1a columnFamilyName b 2024-11-20T20:36:03,628 DEBUG [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:36:03,629 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.HStore(327): Store=6477a8e6578573788f81f020f6167a1a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:36:03,629 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:03,630 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6477a8e6578573788f81f020f6167a1a columnFamilyName c 2024-11-20T20:36:03,630 DEBUG [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:36:03,630 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.HStore(327): Store=6477a8e6578573788f81f020f6167a1a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:36:03,631 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:03,631 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:03,633 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:03,634 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:03,634 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:03,634 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T20:36:03,636 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:03,636 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6477a8e6578573788f81f020f6167a1a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69756594, jitterRate=0.039454251527786255}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T20:36:03,637 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6477a8e6578573788f81f020f6167a1a: Writing region info on filesystem at 1732134963585Initializing all the Stores at 1732134963586 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134963586Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134963586Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134963587 (+1 ms)Cleaning up temporary data from old regions at 1732134963634 (+47 ms)Region opened successfully at 1732134963636 (+2 ms) 2024-11-20T20:36:03,650 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6477a8e6578573788f81f020f6167a1a 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-20T20:36:03,672 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/.tmp/a/fe7323aac9f9420ab5cf325cc55506fb is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732134963637/Put/seqid=0 2024-11-20T20:36:03,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741933_1112 (size=5958) 2024-11-20T20:36:03,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741933_1112 (size=5958) 2024-11-20T20:36:03,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741933_1112 (size=5958) 2024-11-20T20:36:04,090 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/.tmp/a/fe7323aac9f9420ab5cf325cc55506fb 2024-11-20T20:36:04,100 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/.tmp/a/fe7323aac9f9420ab5cf325cc55506fb as hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/a/fe7323aac9f9420ab5cf325cc55506fb 2024-11-20T20:36:04,105 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/a/fe7323aac9f9420ab5cf325cc55506fb, entries=10, sequenceid=13, filesize=5.8 K 2024-11-20T20:36:04,107 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 6477a8e6578573788f81f020f6167a1a in 458ms, sequenceid=13, compaction requested=false 2024-11-20T20:36:04,107 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6477a8e6578573788f81f020f6167a1a: 2024-11-20T20:36:04,124 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6477a8e6578573788f81f020f6167a1a, disabling compactions & flushes 2024-11-20T20:36:04,124 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a. 2024-11-20T20:36:04,124 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a. 2024-11-20T20:36:04,124 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a. after waiting 0 ms 2024-11-20T20:36:04,124 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a. 2024-11-20T20:36:04,125 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a. 2024-11-20T20:36:04,125 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a. 2024-11-20T20:36:04,125 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6477a8e6578573788f81f020f6167a1a: Waiting for close lock at 1732134964124Disabling compacts and flushes for region at 1732134964124Disabling writes for close at 1732134964124Writing region close event to WAL at 1732134964125 (+1 ms)Closed at 1732134964125 2024-11-20T20:36:04,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741932_1111 (size=2805) 2024-11-20T20:36:04,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741932_1111 (size=2805) 2024-11-20T20:36:04,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741932_1111 (size=2805) 2024-11-20T20:36:04,142 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134963540, size=2.7 K (2805bytes) 2024-11-20T20:36:04,142 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134963540 2024-11-20T20:36:04,143 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134963540 after 1ms 2024-11-20T20:36:04,145 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134963540: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:36:04,145 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134963540 took 3ms 2024-11-20T20:36:04,147 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134963540 so closing down 2024-11-20T20:36:04,147 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T20:36:04,149 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732134963540.temp 2024-11-20T20:36:04,150 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000003-wal.1732134963540.temp 2024-11-20T20:36:04,150 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T20:36:04,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741934_1113 (size=2312) 2024-11-20T20:36:04,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741934_1113 (size=2312) 2024-11-20T20:36:04,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741934_1113 (size=2312) 2024-11-20T20:36:04,156 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000003-wal.1732134963540.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-20T20:36:04,158 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000003-wal.1732134963540.temp to hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000035 2024-11-20T20:36:04,158 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134963540, size=2.7 K, length=2805, corrupted=false, cancelled=false 2024-11-20T20:36:04,158 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134963540, journal: Splitting hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134963540, size=2.7 K (2805bytes) at 1732134964142Finishing writing output for hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134963540 so closing down at 1732134964147 (+5 ms)Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000003-wal.1732134963540.temp at 1732134964150 (+3 ms)3 split writer threads finished at 1732134964150Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000003-wal.1732134963540.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732134964156 (+6 ms)Rename recovered edits hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000003-wal.1732134963540.temp to hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000035 at 1732134964158 (+2 ms)Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134963540, size=2.7 K, length=2805, corrupted=false, cancelled=false at 1732134964158 2024-11-20T20:36:04,160 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134963540 to hdfs://localhost:33605/hbase/oldWALs/wal.1732134963540 2024-11-20T20:36:04,161 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000035 2024-11-20T20:36:04,161 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T20:36:04,163 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:36:04,176 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134964163, exclude list is [], retry=0 2024-11-20T20:36:04,179 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:36:04,179 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:36:04,180 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:36:04,184 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134964163 2024-11-20T20:36:04,185 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:38233:38233),(127.0.0.1/127.0.0.1:39759:39759)] 2024-11-20T20:36:04,185 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6477a8e6578573788f81f020f6167a1a, NAME => 'testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a.', STARTKEY => '', ENDKEY => ''} 2024-11-20T20:36:04,185 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:36:04,185 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:04,185 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:04,187 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:04,188 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6477a8e6578573788f81f020f6167a1a columnFamilyName a 2024-11-20T20:36:04,189 DEBUG [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:36:04,198 DEBUG [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/a/fe7323aac9f9420ab5cf325cc55506fb 2024-11-20T20:36:04,198 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.HStore(327): Store=6477a8e6578573788f81f020f6167a1a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:36:04,199 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:04,200 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6477a8e6578573788f81f020f6167a1a columnFamilyName b 2024-11-20T20:36:04,200 DEBUG [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:36:04,200 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.HStore(327): Store=6477a8e6578573788f81f020f6167a1a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:36:04,200 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:04,201 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6477a8e6578573788f81f020f6167a1a columnFamilyName c 2024-11-20T20:36:04,201 DEBUG [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:36:04,202 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.HStore(327): Store=6477a8e6578573788f81f020f6167a1a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:36:04,202 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:04,203 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:04,204 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:04,204 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000035 2024-11-20T20:36:04,206 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:36:04,207 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000035 2024-11-20T20:36:04,208 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6477a8e6578573788f81f020f6167a1a 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-20T20:36:04,222 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/.tmp/b/d7743fa3910e4f999508f0456d9df543 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732134964107/Put/seqid=0 2024-11-20T20:36:04,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741936_1115 (size=5958) 2024-11-20T20:36:04,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741936_1115 (size=5958) 2024-11-20T20:36:04,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741936_1115 (size=5958) 2024-11-20T20:36:04,230 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/.tmp/b/d7743fa3910e4f999508f0456d9df543 2024-11-20T20:36:04,254 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/.tmp/c/886ce48a5e114adea36f6da308cfe6b4 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732134964115/Put/seqid=0 2024-11-20T20:36:04,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741937_1116 (size=5958) 2024-11-20T20:36:04,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741937_1116 (size=5958) 2024-11-20T20:36:04,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741937_1116 (size=5958) 2024-11-20T20:36:04,262 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/.tmp/c/886ce48a5e114adea36f6da308cfe6b4 2024-11-20T20:36:04,269 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/.tmp/b/d7743fa3910e4f999508f0456d9df543 as hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/b/d7743fa3910e4f999508f0456d9df543 2024-11-20T20:36:04,275 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/b/d7743fa3910e4f999508f0456d9df543, entries=10, sequenceid=35, filesize=5.8 K 2024-11-20T20:36:04,276 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/.tmp/c/886ce48a5e114adea36f6da308cfe6b4 as hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/c/886ce48a5e114adea36f6da308cfe6b4 2024-11-20T20:36:04,283 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/c/886ce48a5e114adea36f6da308cfe6b4, entries=10, sequenceid=35, filesize=5.8 K 2024-11-20T20:36:04,283 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 6477a8e6578573788f81f020f6167a1a in 76ms, sequenceid=35, compaction requested=false; wal=null 2024-11-20T20:36:04,284 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000035 2024-11-20T20:36:04,285 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:04,285 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:04,286 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T20:36:04,288 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:04,293 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-20T20:36:04,294 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6477a8e6578573788f81f020f6167a1a; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66998884, jitterRate=-0.0016388297080993652}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T20:36:04,294 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6477a8e6578573788f81f020f6167a1a: Writing region info on filesystem at 1732134964186Initializing all the Stores at 1732134964187 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134964187Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134964187Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134964187Obtaining lock to block concurrent updates at 1732134964208 (+21 ms)Preparing flush snapshotting stores in 6477a8e6578573788f81f020f6167a1a at 1732134964208Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732134964208Flushing stores of testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a. at 1732134964208Flushing 6477a8e6578573788f81f020f6167a1a/b: creating writer at 1732134964208Flushing 6477a8e6578573788f81f020f6167a1a/b: appending metadata at 1732134964222 (+14 ms)Flushing 6477a8e6578573788f81f020f6167a1a/b: closing flushed file at 1732134964222Flushing 6477a8e6578573788f81f020f6167a1a/c: creating writer at 1732134964235 (+13 ms)Flushing 6477a8e6578573788f81f020f6167a1a/c: appending metadata at 1732134964253 (+18 ms)Flushing 6477a8e6578573788f81f020f6167a1a/c: closing flushed file at 1732134964254 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5971cf02: reopening flushed file at 1732134964268 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f9038fd: reopening flushed file at 1732134964275 (+7 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 6477a8e6578573788f81f020f6167a1a in 76ms, sequenceid=35, compaction requested=false; wal=null at 1732134964283 (+8 ms)Cleaning up temporary data from old regions at 1732134964285 (+2 ms)Region opened successfully at 1732134964294 (+9 ms) 2024-11-20T20:36:04,380 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134964163, size=0 (0bytes) 2024-11-20T20:36:04,380 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134964163 might be still open, length is 0 2024-11-20T20:36:04,380 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134964163 2024-11-20T20:36:04,380 WARN [IPC Server handler 1 on default port 33605 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134964163 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-20T20:36:04,380 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134964163 after 0ms 2024-11-20T20:36:05,011 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:50790 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:44519:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50790 dst: /127.0.0.1:44519 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:44519 remote=/127.0.0.1:50790]. Total timeout mills is 60000, 59323 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:36:05,012 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:54402 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:40323:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54402 dst: /127.0.0.1:40323 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:36:05,012 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1420044813_22 at /127.0.0.1:54748 [Receiving block BP-1180650432-172.17.0.2-1732134912528:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:35033:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54748 dst: /127.0.0.1:35033 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:36:05,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741935_1117 (size=2304) 2024-11-20T20:36:05,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741935_1117 (size=2304) 2024-11-20T20:36:08,381 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134964163 after 4001ms 2024-11-20T20:36:08,384 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134964163: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:36:08,384 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134964163 took 4005ms 2024-11-20T20:36:08,390 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134964163; continuing. 2024-11-20T20:36:08,390 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134964163 so closing down 2024-11-20T20:36:08,390 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T20:36:08,392 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732134964163.temp 2024-11-20T20:36:08,393 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000037-wal.1732134964163.temp 2024-11-20T20:36:08,394 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T20:36:08,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741938_1118 (size=2312) 2024-11-20T20:36:08,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741938_1118 (size=2312) 2024-11-20T20:36:08,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741938_1118 (size=2312) 2024-11-20T20:36:08,401 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000037-wal.1732134964163.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-20T20:36:08,403 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000037-wal.1732134964163.temp to hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000066 2024-11-20T20:36:08,403 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134964163, size=0, length=0, corrupted=false, cancelled=false 2024-11-20T20:36:08,404 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134964163, journal: Splitting hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134964163, size=0 (0bytes) at 1732134964380Finishing writing output for hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134964163 so closing down at 1732134968390 (+4010 ms)Creating recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000037-wal.1732134964163.temp at 1732134968393 (+3 ms)3 split writer threads finished at 1732134968394 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000037-wal.1732134964163.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732134968401 (+7 ms)Rename recovered edits hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000037-wal.1732134964163.temp to hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000066 at 1732134968403 (+2 ms)Processed 30 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134964163, size=0, length=0, corrupted=false, cancelled=false at 1732134968404 (+1 ms) 2024-11-20T20:36:08,405 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134964163 to hdfs://localhost:33605/hbase/oldWALs/wal.1732134964163 2024-11-20T20:36:08,406 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000066 2024-11-20T20:36:08,406 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T20:36:08,408 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33605/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408, archiveDir=hdfs://localhost:33605/hbase/oldWALs, maxLogs=32 2024-11-20T20:36:08,415 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-20T20:36:08,415 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-20T20:36:08,416 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-20T20:36:08,417 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-20T20:36:08,422 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134968408, exclude list is [], retry=0 2024-11-20T20:36:08,425 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35033,DS-23666ede-fb32-4ac7-a64c-a6d3c37b1f3a,DISK] 2024-11-20T20:36:08,425 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44519,DS-168a228a-4d47-4268-859c-e933fd2e682c,DISK] 2024-11-20T20:36:08,425 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40323,DS-ed9bfa58-1910-4ce6-8a03-5b850e0ff69c,DISK] 2024-11-20T20:36:08,433 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134968408 2024-11-20T20:36:08,433 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38233:38233),(127.0.0.1/127.0.0.1:46511:46511),(127.0.0.1/127.0.0.1:39759:39759)] 2024-11-20T20:36:08,433 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T20:36:08,435 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:08,436 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6477a8e6578573788f81f020f6167a1a columnFamilyName a 2024-11-20T20:36:08,436 DEBUG [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:36:08,446 DEBUG [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/a/fe7323aac9f9420ab5cf325cc55506fb 2024-11-20T20:36:08,446 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.HStore(327): Store=6477a8e6578573788f81f020f6167a1a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:36:08,446 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:08,447 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6477a8e6578573788f81f020f6167a1a columnFamilyName b 2024-11-20T20:36:08,447 DEBUG [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:36:08,453 DEBUG [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/b/d7743fa3910e4f999508f0456d9df543 2024-11-20T20:36:08,453 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.HStore(327): Store=6477a8e6578573788f81f020f6167a1a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:36:08,453 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:08,454 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6477a8e6578573788f81f020f6167a1a columnFamilyName c 2024-11-20T20:36:08,455 DEBUG [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T20:36:08,460 DEBUG [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/c/886ce48a5e114adea36f6da308cfe6b4 2024-11-20T20:36:08,460 INFO [StoreOpener-6477a8e6578573788f81f020f6167a1a-1 {}] regionserver.HStore(327): Store=6477a8e6578573788f81f020f6167a1a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T20:36:08,461 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:08,462 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:08,463 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:08,464 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000066 2024-11-20T20:36:08,466 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-20T20:36:08,470 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000066 2024-11-20T20:36:08,471 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6477a8e6578573788f81f020f6167a1a 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-20T20:36:08,491 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/.tmp/a/24514c6028b74e2ca5940b08f072daef is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732134964302/Put/seqid=0 2024-11-20T20:36:08,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741940_1120 (size=5958) 2024-11-20T20:36:08,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741940_1120 (size=5958) 2024-11-20T20:36:08,504 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/.tmp/a/24514c6028b74e2ca5940b08f072daef 2024-11-20T20:36:08,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741940_1120 (size=5958) 2024-11-20T20:36:08,534 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/.tmp/b/671a392551174e2fb815c63cbd0b0cf3 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732134964318/Put/seqid=0 2024-11-20T20:36:08,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741941_1121 (size=5958) 2024-11-20T20:36:08,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741941_1121 (size=5958) 2024-11-20T20:36:08,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741941_1121 (size=5958) 2024-11-20T20:36:08,955 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/.tmp/b/671a392551174e2fb815c63cbd0b0cf3 2024-11-20T20:36:08,973 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/.tmp/c/9ea9d37cf809463790b2378f3411f6e8 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732134964327/Put/seqid=0 2024-11-20T20:36:08,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741942_1122 (size=5958) 2024-11-20T20:36:08,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741942_1122 (size=5958) 2024-11-20T20:36:08,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741942_1122 (size=5958) 2024-11-20T20:36:08,980 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/.tmp/c/9ea9d37cf809463790b2378f3411f6e8 2024-11-20T20:36:08,985 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/.tmp/a/24514c6028b74e2ca5940b08f072daef as hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/a/24514c6028b74e2ca5940b08f072daef 2024-11-20T20:36:08,990 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/a/24514c6028b74e2ca5940b08f072daef, entries=10, sequenceid=66, filesize=5.8 K 2024-11-20T20:36:08,991 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/.tmp/b/671a392551174e2fb815c63cbd0b0cf3 as hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/b/671a392551174e2fb815c63cbd0b0cf3 2024-11-20T20:36:08,996 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/b/671a392551174e2fb815c63cbd0b0cf3, entries=10, sequenceid=66, filesize=5.8 K 2024-11-20T20:36:08,997 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/.tmp/c/9ea9d37cf809463790b2378f3411f6e8 as hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/c/9ea9d37cf809463790b2378f3411f6e8 2024-11-20T20:36:09,002 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/c/9ea9d37cf809463790b2378f3411f6e8, entries=10, sequenceid=66, filesize=5.8 K 2024-11-20T20:36:09,002 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 6477a8e6578573788f81f020f6167a1a in 531ms, sequenceid=66, compaction requested=false; wal=null 2024-11-20T20:36:09,003 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/0000000000000000066 2024-11-20T20:36:09,004 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:09,004 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:09,005 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T20:36:09,007 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6477a8e6578573788f81f020f6167a1a 2024-11-20T20:36:09,010 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/hbase/data/default/testReplayEditsWrittenViaHRegion/6477a8e6578573788f81f020f6167a1a/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-20T20:36:09,011 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6477a8e6578573788f81f020f6167a1a; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72945789, jitterRate=0.08697696030139923}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T20:36:09,011 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6477a8e6578573788f81f020f6167a1a: Writing region info on filesystem at 1732134968434Initializing all the Stores at 1732134968435 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134968435Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134968435Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732134968435Obtaining lock to block concurrent updates at 1732134968471 (+36 ms)Preparing flush snapshotting stores in 6477a8e6578573788f81f020f6167a1a at 1732134968471Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732134968471Flushing stores of testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a. at 1732134968471Flushing 6477a8e6578573788f81f020f6167a1a/a: creating writer at 1732134968471Flushing 6477a8e6578573788f81f020f6167a1a/a: appending metadata at 1732134968490 (+19 ms)Flushing 6477a8e6578573788f81f020f6167a1a/a: closing flushed file at 1732134968491 (+1 ms)Flushing 6477a8e6578573788f81f020f6167a1a/b: creating writer at 1732134968513 (+22 ms)Flushing 6477a8e6578573788f81f020f6167a1a/b: appending metadata at 1732134968534 (+21 ms)Flushing 6477a8e6578573788f81f020f6167a1a/b: closing flushed file at 1732134968534Flushing 6477a8e6578573788f81f020f6167a1a/c: creating writer at 1732134968959 (+425 ms)Flushing 6477a8e6578573788f81f020f6167a1a/c: appending metadata at 1732134968972 (+13 ms)Flushing 6477a8e6578573788f81f020f6167a1a/c: closing flushed file at 1732134968972Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@827227b: reopening flushed file at 1732134968984 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fd5c6c1: reopening flushed file at 1732134968990 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d248948: reopening flushed file at 1732134968996 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 6477a8e6578573788f81f020f6167a1a in 531ms, sequenceid=66, compaction requested=false; wal=null at 1732134969002 (+6 ms)Cleaning up temporary data from old regions at 1732134969004 (+2 ms)Region opened successfully at 1732134969011 (+7 ms) 2024-11-20T20:36:09,026 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6477a8e6578573788f81f020f6167a1a, disabling compactions & flushes 2024-11-20T20:36:09,026 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a. 2024-11-20T20:36:09,026 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a. 2024-11-20T20:36:09,026 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a. after waiting 0 ms 2024-11-20T20:36:09,026 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a. 2024-11-20T20:36:09,029 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732134963410.6477a8e6578573788f81f020f6167a1a. 2024-11-20T20:36:09,029 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6477a8e6578573788f81f020f6167a1a: Waiting for close lock at 1732134969026Disabling compacts and flushes for region at 1732134969026Disabling writes for close at 1732134969026Writing region close event to WAL at 1732134969029 (+3 ms)Closed at 1732134969029 2024-11-20T20:36:09,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741939_1119 (size=93) 2024-11-20T20:36:09,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741939_1119 (size=93) 2024-11-20T20:36:09,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741939_1119 (size=93) 2024-11-20T20:36:09,035 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T20:36:09,035 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732134968408) 2024-11-20T20:36:09,050 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=441 (was 434) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_739502407_22 at /127.0.0.1:54762 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_739502407_22 at /127.0.0.1:50848 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:33605 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:33605 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_739502407_22 at /127.0.0.1:54446 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1410 (was 1344) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=825 (was 879), ProcessCount=11 (was 11), AvailableMemoryMB=1402 (was 1431) 2024-11-20T20:36:09,050 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1410 is superior to 1024 2024-11-20T20:36:09,050 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T20:36:09,050 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T20:36:09,051 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T20:36:09,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T20:36:09,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T20:36:09,052 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T20:36:09,052 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T20:36:09,052 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=207465779, stopped=false 2024-11-20T20:36:09,052 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d7d59561a64e,46641,1732134918030 2024-11-20T20:36:09,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36243-0x10159f54fc60003, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T20:36:09,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33553-0x10159f54fc60002, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T20:36:09,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33553-0x10159f54fc60002, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:36:09,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36243-0x10159f54fc60003, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:36:09,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T20:36:09,112 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T20:36:09,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:36:09,112 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T20:36:09,113 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T20:36:09,113 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T20:36:09,113 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd7d59561a64e,33553,1732134919262' ***** 2024-11-20T20:36:09,113 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T20:36:09,113 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd7d59561a64e,36243,1732134919381' ***** 2024-11-20T20:36:09,113 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T20:36:09,114 INFO [RS:1;d7d59561a64e:33553 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T20:36:09,114 INFO [RS:1;d7d59561a64e:33553 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T20:36:09,114 INFO [RS:1;d7d59561a64e:33553 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T20:36:09,114 INFO [RS:1;d7d59561a64e:33553 {}] regionserver.HRegionServer(959): stopping server d7d59561a64e,33553,1732134919262 2024-11-20T20:36:09,114 INFO [RS:1;d7d59561a64e:33553 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T20:36:09,114 INFO [RS:1;d7d59561a64e:33553 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;d7d59561a64e:33553. 2024-11-20T20:36:09,114 DEBUG [RS:1;d7d59561a64e:33553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T20:36:09,114 DEBUG [RS:1;d7d59561a64e:33553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T20:36:09,115 INFO [RS:1;d7d59561a64e:33553 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T20:36:09,115 INFO [RS:1;d7d59561a64e:33553 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T20:36:09,115 INFO [RS:1;d7d59561a64e:33553 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T20:36:09,115 INFO [RS:1;d7d59561a64e:33553 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T20:36:09,115 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T20:36:09,115 INFO [RS:2;d7d59561a64e:36243 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T20:36:09,115 INFO [RS:2;d7d59561a64e:36243 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T20:36:09,115 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T20:36:09,115 INFO [RS:2;d7d59561a64e:36243 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T20:36:09,115 INFO [RS:2;d7d59561a64e:36243 {}] regionserver.HRegionServer(3091): Received CLOSE for e84bafb3668f028684842276e5650d4c 2024-11-20T20:36:09,116 INFO [RS:1;d7d59561a64e:33553 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T20:36:09,116 DEBUG [RS:1;d7d59561a64e:33553 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-20T20:36:09,116 DEBUG [RS:1;d7d59561a64e:33553 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T20:36:09,116 DEBUG [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T20:36:09,116 INFO [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T20:36:09,116 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36243-0x10159f54fc60003, quorum=127.0.0.1:64093, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T20:36:09,116 DEBUG [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T20:36:09,116 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T20:36:09,116 DEBUG [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T20:36:09,116 DEBUG [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T20:36:09,116 INFO [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-20T20:36:09,116 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33553-0x10159f54fc60002, quorum=127.0.0.1:64093, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T20:36:09,118 INFO [RS:2;d7d59561a64e:36243 {}] regionserver.HRegionServer(959): stopping server d7d59561a64e,36243,1732134919381 2024-11-20T20:36:09,119 INFO [RS:2;d7d59561a64e:36243 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T20:36:09,119 INFO [RS:2;d7d59561a64e:36243 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;d7d59561a64e:36243. 2024-11-20T20:36:09,119 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e84bafb3668f028684842276e5650d4c, disabling compactions & flushes 2024-11-20T20:36:09,119 DEBUG [RS:2;d7d59561a64e:36243 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T20:36:09,119 DEBUG [RS:2;d7d59561a64e:36243 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T20:36:09,119 INFO [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:36:09,119 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:36:09,119 INFO [RS:2;d7d59561a64e:36243 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T20:36:09,119 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. after waiting 0 ms 2024-11-20T20:36:09,119 DEBUG [RS:2;d7d59561a64e:36243 {}] regionserver.HRegionServer(1325): Online Regions={e84bafb3668f028684842276e5650d4c=testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c.} 2024-11-20T20:36:09,119 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:36:09,119 DEBUG [RS:2;d7d59561a64e:36243 {}] regionserver.HRegionServer(1351): Waiting on e84bafb3668f028684842276e5650d4c 2024-11-20T20:36:09,131 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/default/testReplayEditsAfterRegionMovedWithMultiCF/e84bafb3668f028684842276e5650d4c/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-20T20:36:09,132 INFO [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:36:09,132 DEBUG [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/.tmp/info/edec15aabe2c46a5968b4b9ced512e6a is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c./info:regioninfo/1732134946160/Put/seqid=0 2024-11-20T20:36:09,132 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e84bafb3668f028684842276e5650d4c: Waiting for close lock at 1732134969119Running coprocessor pre-close hooks at 1732134969119Disabling compacts and flushes for region at 1732134969119Disabling writes for close at 1732134969119Writing region close event to WAL at 1732134969122 (+3 ms)Running coprocessor post-close hooks at 1732134969132 (+10 ms)Closed at 1732134969132 2024-11-20T20:36:09,133 DEBUG [RS_CLOSE_REGION-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732134940363.e84bafb3668f028684842276e5650d4c. 2024-11-20T20:36:09,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741943_1123 (size=6778) 2024-11-20T20:36:09,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741943_1123 (size=6778) 2024-11-20T20:36:09,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741943_1123 (size=6778) 2024-11-20T20:36:09,142 INFO [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/.tmp/info/edec15aabe2c46a5968b4b9ced512e6a 2024-11-20T20:36:09,148 DEBUG [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/.tmp/info/edec15aabe2c46a5968b4b9ced512e6a as hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/info/edec15aabe2c46a5968b4b9ced512e6a 2024-11-20T20:36:09,154 INFO [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/info/edec15aabe2c46a5968b4b9ced512e6a, entries=8, sequenceid=23, filesize=6.6 K 2024-11-20T20:36:09,155 INFO [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 39ms, sequenceid=23, compaction requested=false 2024-11-20T20:36:09,155 DEBUG [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T20:36:09,157 INFO [regionserver/d7d59561a64e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-20T20:36:09,157 INFO [regionserver/d7d59561a64e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-20T20:36:09,161 INFO [regionserver/d7d59561a64e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T20:36:09,163 DEBUG [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-20T20:36:09,164 DEBUG [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T20:36:09,164 INFO [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T20:36:09,164 DEBUG [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732134969116Running coprocessor pre-close hooks at 1732134969116Disabling compacts and flushes for region at 1732134969116Disabling writes for close at 1732134969116Obtaining lock to block concurrent updates at 1732134969116Preparing flush snapshotting stores in 1588230740 at 1732134969116Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1732134969117 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732134969120 (+3 ms)Flushing 1588230740/info: creating writer at 1732134969120Flushing 1588230740/info: appending metadata at 1732134969132 (+12 ms)Flushing 1588230740/info: closing flushed file at 1732134969132Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@651e2b28: reopening flushed file at 1732134969147 (+15 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 39ms, sequenceid=23, compaction requested=false at 1732134969155 (+8 ms)Writing region close event to WAL at 1732134969159 (+4 ms)Running coprocessor post-close hooks at 1732134969164 (+5 ms)Closed at 1732134969164 2024-11-20T20:36:09,165 DEBUG [RS_CLOSE_META-regionserver/d7d59561a64e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T20:36:09,169 INFO [regionserver/d7d59561a64e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T20:36:09,199 INFO [regionserver/d7d59561a64e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-20T20:36:09,199 INFO [regionserver/d7d59561a64e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-20T20:36:09,316 INFO [RS:1;d7d59561a64e:33553 {}] regionserver.HRegionServer(976): stopping server d7d59561a64e,33553,1732134919262; all regions closed. 2024-11-20T20:36:09,319 INFO [RS:2;d7d59561a64e:36243 {}] regionserver.HRegionServer(976): stopping server d7d59561a64e,36243,1732134919381; all regions closed. 2024-11-20T20:36:09,319 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/WALs/d7d59561a64e,33553,1732134919262/d7d59561a64e%2C33553%2C1732134919262.meta.1732134944758.meta not finished, retry = 0 2024-11-20T20:36:09,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741893_1071 (size=1552) 2024-11-20T20:36:09,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741893_1071 (size=1552) 2024-11-20T20:36:09,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741893_1071 (size=1552) 2024-11-20T20:36:09,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741834_1010 (size=841) 2024-11-20T20:36:09,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741834_1010 (size=841) 2024-11-20T20:36:09,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741834_1010 (size=841) 2024-11-20T20:36:09,331 DEBUG [RS:2;d7d59561a64e:36243 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/oldWALs 2024-11-20T20:36:09,331 INFO [RS:2;d7d59561a64e:36243 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d7d59561a64e%2C36243%2C1732134919381:(num 1732134922362) 2024-11-20T20:36:09,331 DEBUG [RS:2;d7d59561a64e:36243 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T20:36:09,331 INFO [RS:2;d7d59561a64e:36243 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T20:36:09,331 INFO [RS:2;d7d59561a64e:36243 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T20:36:09,332 INFO [RS:2;d7d59561a64e:36243 {}] hbase.ChoreService(370): Chore service for: regionserver/d7d59561a64e:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T20:36:09,332 INFO [RS:2;d7d59561a64e:36243 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T20:36:09,332 INFO [regionserver/d7d59561a64e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T20:36:09,332 INFO [RS:2;d7d59561a64e:36243 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T20:36:09,332 INFO [RS:2;d7d59561a64e:36243 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T20:36:09,332 INFO [RS:2;d7d59561a64e:36243 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T20:36:09,332 INFO [RS:2;d7d59561a64e:36243 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36243 2024-11-20T20:36:09,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T20:36:09,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36243-0x10159f54fc60003, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d7d59561a64e,36243,1732134919381 2024-11-20T20:36:09,353 INFO [RS:2;d7d59561a64e:36243 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T20:36:09,364 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d7d59561a64e,36243,1732134919381] 2024-11-20T20:36:09,374 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d7d59561a64e,36243,1732134919381 already deleted, retry=false 2024-11-20T20:36:09,374 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d7d59561a64e,36243,1732134919381 expired; onlineServers=1 2024-11-20T20:36:09,423 DEBUG [RS:1;d7d59561a64e:33553 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/oldWALs 2024-11-20T20:36:09,423 INFO [RS:1;d7d59561a64e:33553 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d7d59561a64e%2C33553%2C1732134919262.meta:.meta(num 1732134944758) 2024-11-20T20:36:09,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741833_1009 (size=2054) 2024-11-20T20:36:09,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741833_1009 (size=2054) 2024-11-20T20:36:09,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741833_1009 (size=2054) 2024-11-20T20:36:09,428 DEBUG [RS:1;d7d59561a64e:33553 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/oldWALs 2024-11-20T20:36:09,428 INFO [RS:1;d7d59561a64e:33553 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d7d59561a64e%2C33553%2C1732134919262:(num 1732134922358) 2024-11-20T20:36:09,428 DEBUG [RS:1;d7d59561a64e:33553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T20:36:09,428 INFO [RS:1;d7d59561a64e:33553 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T20:36:09,428 INFO [RS:1;d7d59561a64e:33553 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T20:36:09,428 INFO [RS:1;d7d59561a64e:33553 {}] hbase.ChoreService(370): Chore service for: regionserver/d7d59561a64e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-20T20:36:09,428 INFO [RS:1;d7d59561a64e:33553 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T20:36:09,428 INFO [regionserver/d7d59561a64e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T20:36:09,428 INFO [RS:1;d7d59561a64e:33553 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33553 2024-11-20T20:36:09,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33553-0x10159f54fc60002, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d7d59561a64e,33553,1732134919262 2024-11-20T20:36:09,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T20:36:09,437 INFO [RS:1;d7d59561a64e:33553 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T20:36:09,448 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d7d59561a64e,33553,1732134919262] 2024-11-20T20:36:09,458 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d7d59561a64e,33553,1732134919262 already deleted, retry=false 2024-11-20T20:36:09,458 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d7d59561a64e,33553,1732134919262 expired; onlineServers=0 2024-11-20T20:36:09,459 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd7d59561a64e,46641,1732134918030' ***** 2024-11-20T20:36:09,459 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T20:36:09,459 INFO [M:0;d7d59561a64e:46641 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T20:36:09,459 INFO [M:0;d7d59561a64e:46641 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T20:36:09,459 DEBUG [M:0;d7d59561a64e:46641 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T20:36:09,459 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T20:36:09,459 DEBUG [M:0;d7d59561a64e:46641 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T20:36:09,459 DEBUG [master/d7d59561a64e:0:becomeActiveMaster-HFileCleaner.small.0-1732134921855 {}] cleaner.HFileCleaner(306): Exit Thread[master/d7d59561a64e:0:becomeActiveMaster-HFileCleaner.small.0-1732134921855,5,FailOnTimeoutGroup] 2024-11-20T20:36:09,459 DEBUG [master/d7d59561a64e:0:becomeActiveMaster-HFileCleaner.large.0-1732134921853 {}] cleaner.HFileCleaner(306): Exit Thread[master/d7d59561a64e:0:becomeActiveMaster-HFileCleaner.large.0-1732134921853,5,FailOnTimeoutGroup] 2024-11-20T20:36:09,460 INFO [M:0;d7d59561a64e:46641 {}] hbase.ChoreService(370): Chore service for: master/d7d59561a64e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T20:36:09,460 INFO [M:0;d7d59561a64e:46641 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T20:36:09,460 DEBUG [M:0;d7d59561a64e:46641 {}] master.HMaster(1795): Stopping service threads 2024-11-20T20:36:09,460 INFO [M:0;d7d59561a64e:46641 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T20:36:09,460 INFO [M:0;d7d59561a64e:46641 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T20:36:09,461 INFO [M:0;d7d59561a64e:46641 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T20:36:09,461 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T20:36:09,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36243-0x10159f54fc60003, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T20:36:09,464 INFO [RS:2;d7d59561a64e:36243 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T20:36:09,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36243-0x10159f54fc60003, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T20:36:09,464 INFO [RS:2;d7d59561a64e:36243 {}] regionserver.HRegionServer(1031): Exiting; stopping=d7d59561a64e,36243,1732134919381; zookeeper connection closed. 2024-11-20T20:36:09,464 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3642fd90 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3642fd90 2024-11-20T20:36:09,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T20:36:09,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T20:36:09,469 DEBUG [M:0;d7d59561a64e:46641 {}] zookeeper.ZKUtil(347): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T20:36:09,469 WARN [M:0;d7d59561a64e:46641 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T20:36:09,470 INFO [M:0;d7d59561a64e:46641 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/.lastflushedseqids 2024-11-20T20:36:09,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741944_1124 (size=119) 2024-11-20T20:36:09,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741944_1124 (size=119) 2024-11-20T20:36:09,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741944_1124 (size=119) 2024-11-20T20:36:09,484 INFO [M:0;d7d59561a64e:46641 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T20:36:09,484 INFO [M:0;d7d59561a64e:46641 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T20:36:09,484 DEBUG [M:0;d7d59561a64e:46641 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T20:36:09,484 INFO [M:0;d7d59561a64e:46641 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T20:36:09,484 DEBUG [M:0;d7d59561a64e:46641 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T20:36:09,484 DEBUG [M:0;d7d59561a64e:46641 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T20:36:09,484 DEBUG [M:0;d7d59561a64e:46641 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T20:36:09,484 INFO [M:0;d7d59561a64e:46641 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.41 KB heapSize=102.71 KB 2024-11-20T20:36:09,500 DEBUG [M:0;d7d59561a64e:46641 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/99e5ecb98318472483219208f8da190d is 82, key is hbase:meta,,1/info:regioninfo/1732134945391/Put/seqid=0 2024-11-20T20:36:09,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741945_1125 (size=6063) 2024-11-20T20:36:09,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741945_1125 (size=6063) 2024-11-20T20:36:09,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741945_1125 (size=6063) 2024-11-20T20:36:09,506 INFO [M:0;d7d59561a64e:46641 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/99e5ecb98318472483219208f8da190d 2024-11-20T20:36:09,545 DEBUG [M:0;d7d59561a64e:46641 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/520f2de1e7bd41149c6eb39d9e8ee005 is 1077, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732134941294/Put/seqid=0 2024-11-20T20:36:09,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33553-0x10159f54fc60002, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T20:36:09,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33553-0x10159f54fc60002, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T20:36:09,551 INFO [RS:1;d7d59561a64e:33553 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T20:36:09,551 INFO [RS:1;d7d59561a64e:33553 {}] regionserver.HRegionServer(1031): Exiting; stopping=d7d59561a64e,33553,1732134919262; zookeeper connection closed. 2024-11-20T20:36:09,553 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@46ba891b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@46ba891b 2024-11-20T20:36:09,553 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-20T20:36:09,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741946_1126 (size=7908) 2024-11-20T20:36:09,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741946_1126 (size=7908) 2024-11-20T20:36:09,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741946_1126 (size=7908) 2024-11-20T20:36:09,565 INFO [M:0;d7d59561a64e:46641 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.18 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/520f2de1e7bd41149c6eb39d9e8ee005 2024-11-20T20:36:09,572 INFO [M:0;d7d59561a64e:46641 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 520f2de1e7bd41149c6eb39d9e8ee005 2024-11-20T20:36:09,600 DEBUG [M:0;d7d59561a64e:46641 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b911640b1d724aa18845f19152058b2e is 69, key is d7d59561a64e,33553,1732134919262/rs:state/1732134921892/Put/seqid=0 2024-11-20T20:36:09,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741947_1127 (size=5440) 2024-11-20T20:36:09,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741947_1127 (size=5440) 2024-11-20T20:36:09,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741947_1127 (size=5440) 2024-11-20T20:36:09,621 INFO [M:0;d7d59561a64e:46641 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b911640b1d724aa18845f19152058b2e 2024-11-20T20:36:09,627 INFO [M:0;d7d59561a64e:46641 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b911640b1d724aa18845f19152058b2e 2024-11-20T20:36:09,628 DEBUG [M:0;d7d59561a64e:46641 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/99e5ecb98318472483219208f8da190d as hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/99e5ecb98318472483219208f8da190d 2024-11-20T20:36:09,634 INFO [M:0;d7d59561a64e:46641 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/99e5ecb98318472483219208f8da190d, entries=14, sequenceid=207, filesize=5.9 K 2024-11-20T20:36:09,636 DEBUG [M:0;d7d59561a64e:46641 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/520f2de1e7bd41149c6eb39d9e8ee005 as hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/520f2de1e7bd41149c6eb39d9e8ee005 2024-11-20T20:36:09,642 INFO [M:0;d7d59561a64e:46641 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 520f2de1e7bd41149c6eb39d9e8ee005 2024-11-20T20:36:09,642 INFO [M:0;d7d59561a64e:46641 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/520f2de1e7bd41149c6eb39d9e8ee005, entries=21, sequenceid=207, filesize=7.7 K 2024-11-20T20:36:09,643 DEBUG [M:0;d7d59561a64e:46641 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b911640b1d724aa18845f19152058b2e as hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b911640b1d724aa18845f19152058b2e 2024-11-20T20:36:09,650 INFO [M:0;d7d59561a64e:46641 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b911640b1d724aa18845f19152058b2e 2024-11-20T20:36:09,650 INFO [M:0;d7d59561a64e:46641 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33605/user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b911640b1d724aa18845f19152058b2e, entries=3, sequenceid=207, filesize=5.3 K 2024-11-20T20:36:09,657 INFO [M:0;d7d59561a64e:46641 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.41 KB/85410, heapSize ~102.41 KB/104872, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 172ms, sequenceid=207, compaction requested=false 2024-11-20T20:36:09,664 INFO [M:0;d7d59561a64e:46641 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T20:36:09,665 DEBUG [M:0;d7d59561a64e:46641 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732134969484Disabling compacts and flushes for region at 1732134969484Disabling writes for close at 1732134969484Obtaining lock to block concurrent updates at 1732134969484Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732134969484Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85410, getHeapSize=105112, getOffHeapSize=0, getCellsCount=248 at 1732134969485 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732134969485Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732134969485Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732134969499 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732134969499Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732134969511 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732134969545 (+34 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732134969545Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732134969572 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732134969596 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732134969596Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@428bd322: reopening flushed file at 1732134969627 (+31 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b884aae: reopening flushed file at 1732134969635 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54c44bec: reopening flushed file at 1732134969642 (+7 ms)Finished flush of dataSize ~83.41 KB/85410, heapSize ~102.41 KB/104872, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 172ms, sequenceid=207, compaction requested=false at 1732134969657 (+15 ms)Writing region close event to WAL at 1732134969664 (+7 ms)Closed at 1732134969664 2024-11-20T20:36:09,670 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/39c3bfc5-6949-c9ba-6f04-de1132e81fb6/MasterData/WALs/d7d59561a64e,46641,1732134918030/d7d59561a64e%2C46641%2C1732134918030.1732134920771 not finished, retry = 0 2024-11-20T20:36:09,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40323 is added to blk_1073741830_1006 (size=86759) 2024-11-20T20:36:09,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35033 is added to blk_1073741830_1006 (size=86759) 2024-11-20T20:36:09,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44519 is added to blk_1073741830_1006 (size=86759) 2024-11-20T20:36:09,772 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T20:36:09,772 INFO [M:0;d7d59561a64e:46641 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T20:36:09,772 INFO [M:0;d7d59561a64e:46641 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46641 2024-11-20T20:36:09,772 INFO [M:0;d7d59561a64e:46641 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T20:36:09,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T20:36:09,885 INFO [M:0;d7d59561a64e:46641 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T20:36:09,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46641-0x10159f54fc60000, quorum=127.0.0.1:64093, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T20:36:09,897 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732134947961/wal.1732134957405 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732134947961/wal.1732134957405 (inode 16688) Holder DFSClient_NONMAPREDUCE_-1420044813_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732134947961/wal.1732134957405 (inode 16688) Holder DFSClient_NONMAPREDUCE_-1420044813_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-20T20:36:09,900 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732134946586/wal.1732134947675 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732134946586/wal.1732134947675 (inode 16665) Holder DFSClient_NONMAPREDUCE_-1420044813_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732134946586/wal.1732134947675 (inode 16665) Holder DFSClient_NONMAPREDUCE_-1420044813_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-20T20:36:09,901 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732134935707/wal.1732134936012 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:36:09,903 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732134957623/wal.1732134958363 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732134957623/wal.1732134958363 (inode 16714) Holder DFSClient_NONMAPREDUCE_-1420044813_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732134957623/wal.1732134958363 (inode 16714) Holder DFSClient_NONMAPREDUCE_-1420044813_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-20T20:36:09,904 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732134958922/wal.1732134958979 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:36:09,904 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732134963408/wal.1732134964163 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:36:09,904 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732134924726/wal.1732134924971 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T20:36:09,906 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal.1732134924570 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal.1732134924570 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1420044813_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732134924233/wal.1732134924570 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1420044813_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-20T20:36:09,910 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal.1732134923982 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal.1732134923982 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1420044813_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732134923604/wal.1732134923982 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1420044813_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-20T20:36:09,914 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4290616c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T20:36:09,916 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5e25d2b6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T20:36:09,916 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T20:36:09,916 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6853f5bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T20:36:09,917 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1bf0fdb4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/hadoop.log.dir/,STOPPED} 2024-11-20T20:36:09,919 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T20:36:09,919 WARN [BP-1180650432-172.17.0.2-1732134912528 heartbeating to localhost/127.0.0.1:33605 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T20:36:09,919 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T20:36:09,919 WARN [BP-1180650432-172.17.0.2-1732134912528 heartbeating to localhost/127.0.0.1:33605 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1180650432-172.17.0.2-1732134912528 (Datanode Uuid 16edd396-61c9-42d4-ae15-5798a67a52d9) service to localhost/127.0.0.1:33605 2024-11-20T20:36:09,920 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/cluster_bbcbe4e8-087e-d341-dffc-b7e2617acb01/data/data5/current/BP-1180650432-172.17.0.2-1732134912528 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T20:36:09,920 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/cluster_bbcbe4e8-087e-d341-dffc-b7e2617acb01/data/data6/current/BP-1180650432-172.17.0.2-1732134912528 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T20:36:09,921 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T20:36:09,923 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@28e1ba78{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T20:36:09,923 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@702c0733{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T20:36:09,923 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T20:36:09,923 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3264f7d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T20:36:09,923 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e971547{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/hadoop.log.dir/,STOPPED} 2024-11-20T20:36:09,924 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T20:36:09,924 WARN [BP-1180650432-172.17.0.2-1732134912528 heartbeating to localhost/127.0.0.1:33605 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T20:36:09,924 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T20:36:09,924 WARN [BP-1180650432-172.17.0.2-1732134912528 heartbeating to localhost/127.0.0.1:33605 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1180650432-172.17.0.2-1732134912528 (Datanode Uuid 9bc160bb-b0dc-4c50-97e0-2f3d53833c72) service to localhost/127.0.0.1:33605 2024-11-20T20:36:09,925 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/cluster_bbcbe4e8-087e-d341-dffc-b7e2617acb01/data/data3/current/BP-1180650432-172.17.0.2-1732134912528 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T20:36:09,925 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/cluster_bbcbe4e8-087e-d341-dffc-b7e2617acb01/data/data4/current/BP-1180650432-172.17.0.2-1732134912528 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T20:36:09,925 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T20:36:09,927 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4dc44ec6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T20:36:09,927 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@17f6e478{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T20:36:09,927 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T20:36:09,927 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3d33512e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T20:36:09,928 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b895d95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/hadoop.log.dir/,STOPPED} 2024-11-20T20:36:09,928 WARN [BP-1180650432-172.17.0.2-1732134912528 heartbeating to localhost/127.0.0.1:33605 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T20:36:09,928 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T20:36:09,928 WARN [BP-1180650432-172.17.0.2-1732134912528 heartbeating to localhost/127.0.0.1:33605 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1180650432-172.17.0.2-1732134912528 (Datanode Uuid 6beff4b0-2a83-454d-9cae-0ce8f6e616ce) service to localhost/127.0.0.1:33605 2024-11-20T20:36:09,928 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T20:36:09,929 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/cluster_bbcbe4e8-087e-d341-dffc-b7e2617acb01/data/data1/current/BP-1180650432-172.17.0.2-1732134912528 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T20:36:09,929 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/cluster_bbcbe4e8-087e-d341-dffc-b7e2617acb01/data/data2/current/BP-1180650432-172.17.0.2-1732134912528 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T20:36:09,929 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T20:36:09,936 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1843526c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T20:36:09,936 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@776c4d96{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T20:36:09,937 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T20:36:09,937 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70596a48{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T20:36:09,937 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6630be9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/44d027db-35a0-cdda-26bc-8d2f24de7986/hadoop.log.dir/,STOPPED} 2024-11-20T20:36:09,949 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T20:36:10,007 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down