2024-12-09 08:19:05,233 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-09 08:19:05,243 main DEBUG Took 0.008693 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 08:19:05,244 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 08:19:05,244 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 08:19:05,245 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 08:19:05,246 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:19:05,252 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 08:19:05,263 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:19:05,264 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:19:05,264 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:19:05,265 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:19:05,265 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:19:05,266 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:19:05,266 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:19:05,267 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:19:05,267 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:19:05,267 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:19:05,268 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:19:05,268 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:19:05,269 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:19:05,269 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:19:05,269 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:19:05,270 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:19:05,270 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:19:05,270 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:19:05,271 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:19:05,271 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:19:05,271 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:19:05,271 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:19:05,272 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:19:05,272 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 08:19:05,272 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:19:05,273 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 08:19:05,274 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 08:19:05,275 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 08:19:05,277 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 08:19:05,277 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 08:19:05,279 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 08:19:05,279 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 08:19:05,287 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 08:19:05,289 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 08:19:05,291 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 08:19:05,291 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 08:19:05,292 main DEBUG createAppenders(={Console}) 2024-12-09 08:19:05,293 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-09 08:19:05,293 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-09 08:19:05,293 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-09 08:19:05,294 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 08:19:05,294 main DEBUG OutputStream closed 2024-12-09 08:19:05,294 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 08:19:05,295 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 08:19:05,295 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-09 08:19:05,358 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 08:19:05,360 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 08:19:05,361 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 08:19:05,362 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 08:19:05,363 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 08:19:05,363 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 08:19:05,364 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 08:19:05,364 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 08:19:05,364 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 08:19:05,364 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 08:19:05,365 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 08:19:05,365 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 08:19:05,365 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 08:19:05,365 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 08:19:05,366 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 08:19:05,366 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 08:19:05,366 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 08:19:05,367 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 08:19:05,369 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 08:19:05,369 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-09 08:19:05,370 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 08:19:05,370 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-09T08:19:05,566 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd 2024-12-09 08:19:05,568 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 08:19:05,569 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T08:19:05,576 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-09T08:19:05,581 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-12-09T08:19:05,600 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-09T08:19:05,640 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-09T08:19:05,640 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-09T08:19:05,651 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-09T08:19:05,663 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/cluster_95cc11f3-affe-6e07-bfcd-38f5721bf4f5, deleteOnExit=true 2024-12-09T08:19:05,663 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-09T08:19:05,664 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/test.cache.data in system properties and HBase conf 2024-12-09T08:19:05,665 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/hadoop.tmp.dir in system properties and HBase conf 2024-12-09T08:19:05,665 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/hadoop.log.dir in system properties and HBase conf 2024-12-09T08:19:05,665 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-09T08:19:05,666 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-09T08:19:05,666 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-09T08:19:05,734 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T08:19:05,830 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T08:19:05,835 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-09T08:19:05,835 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-09T08:19:05,836 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-09T08:19:05,837 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T08:19:05,837 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-09T08:19:05,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-09T08:19:05,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T08:19:05,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T08:19:05,840 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-09T08:19:05,840 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/nfs.dump.dir in system properties and HBase conf 2024-12-09T08:19:05,841 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/java.io.tmpdir in system properties and HBase conf 2024-12-09T08:19:05,841 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T08:19:05,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-09T08:19:05,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-09T08:19:06,873 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-09T08:19:06,938 INFO [Time-limited test {}] log.Log(170): Logging initialized @2309ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-09T08:19:07,005 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T08:19:07,060 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T08:19:07,077 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T08:19:07,077 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T08:19:07,078 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T08:19:07,101 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T08:19:07,104 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/hadoop.log.dir/,AVAILABLE} 2024-12-09T08:19:07,106 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T08:19:07,268 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c03e044{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/java.io.tmpdir/jetty-localhost-34833-hadoop-hdfs-3_4_1-tests_jar-_-any-6582698513815580568/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T08:19:07,275 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:34833} 2024-12-09T08:19:07,275 INFO [Time-limited test {}] server.Server(415): Started @2646ms 2024-12-09T08:19:07,801 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T08:19:07,808 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T08:19:07,809 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T08:19:07,809 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T08:19:07,809 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-09T08:19:07,810 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/hadoop.log.dir/,AVAILABLE} 2024-12-09T08:19:07,810 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T08:19:07,905 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@501095cc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/java.io.tmpdir/jetty-localhost-34313-hadoop-hdfs-3_4_1-tests_jar-_-any-10416569490038644570/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T08:19:07,905 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:34313} 2024-12-09T08:19:07,906 INFO [Time-limited test {}] server.Server(415): Started @3276ms 2024-12-09T08:19:07,951 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T08:19:08,048 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T08:19:08,053 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T08:19:08,055 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T08:19:08,055 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T08:19:08,055 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-09T08:19:08,056 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/hadoop.log.dir/,AVAILABLE} 2024-12-09T08:19:08,057 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T08:19:08,170 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64052af6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/java.io.tmpdir/jetty-localhost-34529-hadoop-hdfs-3_4_1-tests_jar-_-any-12297168030101196539/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T08:19:08,171 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:34529} 2024-12-09T08:19:08,171 INFO [Time-limited test {}] server.Server(415): Started @3541ms 2024-12-09T08:19:08,173 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T08:19:08,204 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T08:19:08,210 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T08:19:08,211 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T08:19:08,211 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T08:19:08,212 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T08:19:08,213 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/hadoop.log.dir/,AVAILABLE} 2024-12-09T08:19:08,213 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T08:19:08,310 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@420bc738{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/java.io.tmpdir/jetty-localhost-35577-hadoop-hdfs-3_4_1-tests_jar-_-any-3050576694609112648/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T08:19:08,311 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:35577} 2024-12-09T08:19:08,311 INFO [Time-limited test {}] server.Server(415): Started @3682ms 2024-12-09T08:19:08,314 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T08:19:09,799 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/cluster_95cc11f3-affe-6e07-bfcd-38f5721bf4f5/data/data4/current/BP-1712704723-172.17.0.2-1733732346335/current, will proceed with Du for space computation calculation, 2024-12-09T08:19:09,799 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/cluster_95cc11f3-affe-6e07-bfcd-38f5721bf4f5/data/data3/current/BP-1712704723-172.17.0.2-1733732346335/current, will proceed with Du for space computation calculation, 2024-12-09T08:19:09,799 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/cluster_95cc11f3-affe-6e07-bfcd-38f5721bf4f5/data/data2/current/BP-1712704723-172.17.0.2-1733732346335/current, will proceed with Du for space computation calculation, 2024-12-09T08:19:09,799 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/cluster_95cc11f3-affe-6e07-bfcd-38f5721bf4f5/data/data1/current/BP-1712704723-172.17.0.2-1733732346335/current, will proceed with Du for space computation calculation, 2024-12-09T08:19:09,830 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T08:19:09,830 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T08:19:09,879 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe680ec39406f1252 with lease ID 0x2580dbe6bbb4745e: Processing first storage report for DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10 from datanode DatanodeRegistration(127.0.0.1:38689, datanodeUuid=40980e3f-e063-4027-a6f2-2240b9222764, infoPort=42683, infoSecurePort=0, ipcPort=42153, storageInfo=lv=-57;cid=testClusterID;nsid=1496061186;c=1733732346335) 2024-12-09T08:19:09,881 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe680ec39406f1252 with lease ID 0x2580dbe6bbb4745e: from storage DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10 node DatanodeRegistration(127.0.0.1:38689, datanodeUuid=40980e3f-e063-4027-a6f2-2240b9222764, infoPort=42683, infoSecurePort=0, ipcPort=42153, storageInfo=lv=-57;cid=testClusterID;nsid=1496061186;c=1733732346335), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-09T08:19:09,881 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x41ea209b8e13b1fe with lease ID 0x2580dbe6bbb4745d: Processing first storage report for DS-5e413aed-71fe-4f71-a7b3-1219b426abcf from datanode DatanodeRegistration(127.0.0.1:45463, datanodeUuid=8817ac4d-f808-4e08-9488-ec5e11a1e412, infoPort=41283, infoSecurePort=0, ipcPort=34795, storageInfo=lv=-57;cid=testClusterID;nsid=1496061186;c=1733732346335) 2024-12-09T08:19:09,882 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x41ea209b8e13b1fe with lease ID 0x2580dbe6bbb4745d: from storage DS-5e413aed-71fe-4f71-a7b3-1219b426abcf node DatanodeRegistration(127.0.0.1:45463, datanodeUuid=8817ac4d-f808-4e08-9488-ec5e11a1e412, infoPort=41283, infoSecurePort=0, ipcPort=34795, storageInfo=lv=-57;cid=testClusterID;nsid=1496061186;c=1733732346335), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-09T08:19:09,882 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe680ec39406f1252 with lease ID 0x2580dbe6bbb4745e: Processing first storage report for DS-56e1130f-e2c9-41d9-9d2d-46a64c1f6f53 from datanode DatanodeRegistration(127.0.0.1:38689, datanodeUuid=40980e3f-e063-4027-a6f2-2240b9222764, infoPort=42683, infoSecurePort=0, ipcPort=42153, storageInfo=lv=-57;cid=testClusterID;nsid=1496061186;c=1733732346335) 2024-12-09T08:19:09,882 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe680ec39406f1252 with lease ID 0x2580dbe6bbb4745e: from storage DS-56e1130f-e2c9-41d9-9d2d-46a64c1f6f53 node DatanodeRegistration(127.0.0.1:38689, datanodeUuid=40980e3f-e063-4027-a6f2-2240b9222764, infoPort=42683, infoSecurePort=0, ipcPort=42153, storageInfo=lv=-57;cid=testClusterID;nsid=1496061186;c=1733732346335), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T08:19:09,883 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x41ea209b8e13b1fe with lease ID 0x2580dbe6bbb4745d: Processing first storage report for DS-e9585bb0-6187-4473-bc5f-060f471e111e from datanode DatanodeRegistration(127.0.0.1:45463, datanodeUuid=8817ac4d-f808-4e08-9488-ec5e11a1e412, infoPort=41283, infoSecurePort=0, ipcPort=34795, storageInfo=lv=-57;cid=testClusterID;nsid=1496061186;c=1733732346335) 2024-12-09T08:19:09,883 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x41ea209b8e13b1fe with lease ID 0x2580dbe6bbb4745d: from storage DS-e9585bb0-6187-4473-bc5f-060f471e111e node DatanodeRegistration(127.0.0.1:45463, datanodeUuid=8817ac4d-f808-4e08-9488-ec5e11a1e412, infoPort=41283, infoSecurePort=0, ipcPort=34795, storageInfo=lv=-57;cid=testClusterID;nsid=1496061186;c=1733732346335), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T08:19:09,887 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/cluster_95cc11f3-affe-6e07-bfcd-38f5721bf4f5/data/data5/current/BP-1712704723-172.17.0.2-1733732346335/current, will proceed with Du for space computation calculation, 2024-12-09T08:19:09,887 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/cluster_95cc11f3-affe-6e07-bfcd-38f5721bf4f5/data/data6/current/BP-1712704723-172.17.0.2-1733732346335/current, will proceed with Du for space computation calculation, 2024-12-09T08:19:09,916 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T08:19:09,921 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcd618947c4d3777e with lease ID 0x2580dbe6bbb4745f: Processing first storage report for DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195 from datanode DatanodeRegistration(127.0.0.1:41399, datanodeUuid=d6a66b69-c727-452f-a95a-1253d0f4e848, infoPort=42925, infoSecurePort=0, ipcPort=45559, storageInfo=lv=-57;cid=testClusterID;nsid=1496061186;c=1733732346335) 2024-12-09T08:19:09,921 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcd618947c4d3777e with lease ID 0x2580dbe6bbb4745f: from storage DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195 node DatanodeRegistration(127.0.0.1:41399, datanodeUuid=d6a66b69-c727-452f-a95a-1253d0f4e848, infoPort=42925, infoSecurePort=0, ipcPort=45559, storageInfo=lv=-57;cid=testClusterID;nsid=1496061186;c=1733732346335), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T08:19:09,922 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcd618947c4d3777e with lease ID 0x2580dbe6bbb4745f: Processing first storage report for DS-282007c3-4fb1-4b4d-a8ae-b3adad1355df from datanode DatanodeRegistration(127.0.0.1:41399, datanodeUuid=d6a66b69-c727-452f-a95a-1253d0f4e848, infoPort=42925, infoSecurePort=0, ipcPort=45559, storageInfo=lv=-57;cid=testClusterID;nsid=1496061186;c=1733732346335) 2024-12-09T08:19:09,922 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcd618947c4d3777e with lease ID 0x2580dbe6bbb4745f: from storage DS-282007c3-4fb1-4b4d-a8ae-b3adad1355df node DatanodeRegistration(127.0.0.1:41399, datanodeUuid=d6a66b69-c727-452f-a95a-1253d0f4e848, infoPort=42925, infoSecurePort=0, ipcPort=45559, storageInfo=lv=-57;cid=testClusterID;nsid=1496061186;c=1733732346335), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-09T08:19:09,930 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd 2024-12-09T08:19:09,996 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/cluster_95cc11f3-affe-6e07-bfcd-38f5721bf4f5/zookeeper_0, clientPort=55499, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/cluster_95cc11f3-affe-6e07-bfcd-38f5721bf4f5/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/cluster_95cc11f3-affe-6e07-bfcd-38f5721bf4f5/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T08:19:10,005 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55499 2024-12-09T08:19:10,016 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:19:10,018 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:19:10,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741825_1001 (size=7) 2024-12-09T08:19:10,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741825_1001 (size=7) 2024-12-09T08:19:10,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741825_1001 (size=7) 2024-12-09T08:19:10,600 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3 with version=8 2024-12-09T08:19:10,600 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/hbase-staging 2024-12-09T08:19:10,876 INFO [Time-limited test {}] client.ConnectionUtils(128): master/5bda946059d1:0 server-side Connection retries=45 2024-12-09T08:19:10,884 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T08:19:10,885 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T08:19:10,889 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T08:19:10,889 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T08:19:10,889 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T08:19:11,009 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-09T08:19:11,071 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T08:19:11,083 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T08:19:11,088 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T08:19:11,125 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 55844 (auto-detected) 2024-12-09T08:19:11,127 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-09T08:19:11,152 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41647 2024-12-09T08:19:11,171 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41647 connecting to ZooKeeper ensemble=127.0.0.1:55499 2024-12-09T08:19:11,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:416470x0, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T08:19:11,297 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41647-0x10009a83ed90000 connected 2024-12-09T08:19:11,376 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:19:11,382 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:19:11,392 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T08:19:11,395 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3, hbase.cluster.distributed=false 2024-12-09T08:19:11,418 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T08:19:11,422 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41647 2024-12-09T08:19:11,422 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41647 2024-12-09T08:19:11,423 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41647 2024-12-09T08:19:11,424 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41647 2024-12-09T08:19:11,424 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41647 2024-12-09T08:19:11,513 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5bda946059d1:0 server-side Connection retries=45 2024-12-09T08:19:11,514 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T08:19:11,514 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T08:19:11,515 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T08:19:11,515 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T08:19:11,515 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T08:19:11,517 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T08:19:11,519 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T08:19:11,520 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33313 2024-12-09T08:19:11,522 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33313 connecting to ZooKeeper ensemble=127.0.0.1:55499 2024-12-09T08:19:11,523 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:19:11,525 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:19:11,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:333130x0, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T08:19:11,541 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:333130x0, quorum=127.0.0.1:55499, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T08:19:11,541 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33313-0x10009a83ed90001 connected 2024-12-09T08:19:11,545 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T08:19:11,552 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T08:19:11,555 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33313-0x10009a83ed90001, quorum=127.0.0.1:55499, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T08:19:11,560 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33313-0x10009a83ed90001, quorum=127.0.0.1:55499, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T08:19:11,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33313 2024-12-09T08:19:11,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33313 2024-12-09T08:19:11,562 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33313 2024-12-09T08:19:11,563 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33313 2024-12-09T08:19:11,563 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33313 2024-12-09T08:19:11,577 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5bda946059d1:0 server-side Connection retries=45 2024-12-09T08:19:11,577 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T08:19:11,577 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T08:19:11,577 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T08:19:11,578 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T08:19:11,578 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T08:19:11,578 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T08:19:11,578 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T08:19:11,579 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34083 2024-12-09T08:19:11,581 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34083 connecting to ZooKeeper ensemble=127.0.0.1:55499 2024-12-09T08:19:11,582 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:19:11,585 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:19:11,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:340830x0, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T08:19:11,595 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34083-0x10009a83ed90002 connected 2024-12-09T08:19:11,595 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34083-0x10009a83ed90002, quorum=127.0.0.1:55499, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T08:19:11,596 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T08:19:11,597 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T08:19:11,598 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34083-0x10009a83ed90002, quorum=127.0.0.1:55499, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T08:19:11,600 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34083-0x10009a83ed90002, quorum=127.0.0.1:55499, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T08:19:11,603 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34083 2024-12-09T08:19:11,603 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34083 2024-12-09T08:19:11,604 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34083 2024-12-09T08:19:11,605 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34083 2024-12-09T08:19:11,605 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34083 2024-12-09T08:19:11,620 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5bda946059d1:0 server-side Connection retries=45 2024-12-09T08:19:11,620 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T08:19:11,620 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T08:19:11,620 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T08:19:11,620 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T08:19:11,620 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T08:19:11,621 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T08:19:11,621 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T08:19:11,622 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39779 2024-12-09T08:19:11,623 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39779 connecting to ZooKeeper ensemble=127.0.0.1:55499 2024-12-09T08:19:11,624 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:19:11,627 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:19:11,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:397790x0, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T08:19:11,638 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39779-0x10009a83ed90003 connected 2024-12-09T08:19:11,638 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:397790x0, quorum=127.0.0.1:55499, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T08:19:11,638 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T08:19:11,639 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T08:19:11,640 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39779-0x10009a83ed90003, quorum=127.0.0.1:55499, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T08:19:11,642 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39779-0x10009a83ed90003, quorum=127.0.0.1:55499, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T08:19:11,643 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39779 2024-12-09T08:19:11,643 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39779 2024-12-09T08:19:11,644 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39779 2024-12-09T08:19:11,644 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39779 2024-12-09T08:19:11,644 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39779 2024-12-09T08:19:11,658 DEBUG [M:0;5bda946059d1:41647 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;5bda946059d1:41647 2024-12-09T08:19:11,659 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/5bda946059d1,41647,1733732350732 2024-12-09T08:19:11,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33313-0x10009a83ed90001, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T08:19:11,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34083-0x10009a83ed90002, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T08:19:11,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10009a83ed90003, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T08:19:11,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T08:19:11,671 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/5bda946059d1,41647,1733732350732 2024-12-09T08:19:11,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34083-0x10009a83ed90002, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T08:19:11,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10009a83ed90003, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T08:19:11,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33313-0x10009a83ed90001, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T08:19:11,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:11,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34083-0x10009a83ed90002, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:11,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10009a83ed90003, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:11,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33313-0x10009a83ed90001, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:11,702 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T08:19:11,703 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/5bda946059d1,41647,1733732350732 from backup master directory 2024-12-09T08:19:11,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/5bda946059d1,41647,1733732350732 2024-12-09T08:19:11,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10009a83ed90003, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T08:19:11,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34083-0x10009a83ed90002, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T08:19:11,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33313-0x10009a83ed90001, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T08:19:11,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T08:19:11,712 WARN [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T08:19:11,712 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=5bda946059d1,41647,1733732350732 2024-12-09T08:19:11,715 INFO [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-09T08:19:11,717 INFO [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-09T08:19:11,767 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/hbase.id] with ID: 21dc23ac-0dff-402b-b794-53b2dfbe03d4 2024-12-09T08:19:11,767 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/.tmp/hbase.id 2024-12-09T08:19:11,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741826_1002 (size=42) 2024-12-09T08:19:11,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741826_1002 (size=42) 2024-12-09T08:19:11,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741826_1002 (size=42) 2024-12-09T08:19:11,782 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/.tmp/hbase.id]:[hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/hbase.id] 2024-12-09T08:19:11,819 INFO [master/5bda946059d1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T08:19:11,825 INFO [master/5bda946059d1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-09T08:19:11,842 INFO [master/5bda946059d1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-09T08:19:11,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34083-0x10009a83ed90002, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:11,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10009a83ed90003, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:11,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:11,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33313-0x10009a83ed90001, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:11,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741827_1003 (size=196) 2024-12-09T08:19:11,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741827_1003 (size=196) 2024-12-09T08:19:11,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741827_1003 (size=196) 2024-12-09T08:19:11,890 INFO [master/5bda946059d1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T08:19:11,891 INFO [master/5bda946059d1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T08:19:11,896 INFO [master/5bda946059d1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:11,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741828_1004 (size=1189) 2024-12-09T08:19:11,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741828_1004 (size=1189) 2024-12-09T08:19:11,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741828_1004 (size=1189) 2024-12-09T08:19:11,943 INFO [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/MasterData/data/master/store 2024-12-09T08:19:11,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741829_1005 (size=34) 2024-12-09T08:19:11,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741829_1005 (size=34) 2024-12-09T08:19:11,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741829_1005 (size=34) 2024-12-09T08:19:11,967 INFO [master/5bda946059d1:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T08:19:11,970 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:11,971 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T08:19:11,971 INFO [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T08:19:11,972 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T08:19:11,973 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T08:19:11,973 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T08:19:11,973 INFO [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T08:19:11,974 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733732351971Disabling compacts and flushes for region at 1733732351971Disabling writes for close at 1733732351973 (+2 ms)Writing region close event to WAL at 1733732351973Closed at 1733732351973 2024-12-09T08:19:11,976 WARN [master/5bda946059d1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/MasterData/data/master/store/.initializing 2024-12-09T08:19:11,976 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/MasterData/WALs/5bda946059d1,41647,1733732350732 2024-12-09T08:19:11,983 INFO [master/5bda946059d1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:19:11,999 INFO [master/5bda946059d1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5bda946059d1%2C41647%2C1733732350732, suffix=, logDir=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/MasterData/WALs/5bda946059d1,41647,1733732350732, archiveDir=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/MasterData/oldWALs, maxLogs=10 2024-12-09T08:19:12,035 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/MasterData/WALs/5bda946059d1,41647,1733732350732/5bda946059d1%2C41647%2C1733732350732.1733732352004, exclude list is [], retry=0 2024-12-09T08:19:12,051 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:12,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:12,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:12,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:12,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-09T08:19:12,087 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-12-09T08:19:12,139 INFO [master/5bda946059d1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/MasterData/WALs/5bda946059d1,41647,1733732350732/5bda946059d1%2C41647%2C1733732350732.1733732352004 2024-12-09T08:19:12,140 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41283:41283),(127.0.0.1/127.0.0.1:42925:42925),(127.0.0.1/127.0.0.1:42683:42683)] 2024-12-09T08:19:12,140 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:19:12,140 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:12,143 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:19:12,144 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:19:12,174 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:19:12,194 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T08:19:12,196 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:12,199 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:19:12,199 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:19:12,202 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T08:19:12,202 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:12,203 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:12,203 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:19:12,206 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T08:19:12,206 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:12,207 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:12,207 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:19:12,209 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T08:19:12,209 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:12,210 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:12,210 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:19:12,213 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:19:12,214 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:19:12,219 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:19:12,219 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:19:12,222 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T08:19:12,226 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T08:19:12,229 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:19:12,230 INFO [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71345334, jitterRate=0.06312832236289978}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T08:19:12,236 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733732352154Initializing all the Stores at 1733732352156 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733732352157 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732352157Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732352158 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732352158Cleaning up temporary data from old regions at 1733732352219 (+61 ms)Region opened successfully at 1733732352236 (+17 ms) 2024-12-09T08:19:12,237 INFO [master/5bda946059d1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T08:19:12,265 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43796d5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5bda946059d1/172.17.0.2:0 2024-12-09T08:19:12,290 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-09T08:19:12,299 INFO [master/5bda946059d1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-09T08:19:12,299 INFO [master/5bda946059d1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-09T08:19:12,302 INFO [master/5bda946059d1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T08:19:12,303 INFO [master/5bda946059d1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-09T08:19:12,307 INFO [master/5bda946059d1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-09T08:19:12,307 INFO [master/5bda946059d1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T08:19:12,328 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T08:19:12,336 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T08:19:12,384 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-09T08:19:12,388 INFO [master/5bda946059d1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T08:19:12,390 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T08:19:12,403 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-09T08:19:12,405 INFO [master/5bda946059d1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T08:19:12,408 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T08:19:12,415 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-09T08:19:12,417 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T08:19:12,426 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T08:19:12,446 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T08:19:12,457 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T08:19:12,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10009a83ed90003, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T08:19:12,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33313-0x10009a83ed90001, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T08:19:12,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34083-0x10009a83ed90002, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T08:19:12,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T08:19:12,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34083-0x10009a83ed90002, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:12,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:12,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10009a83ed90003, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:12,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33313-0x10009a83ed90001, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:12,474 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=5bda946059d1,41647,1733732350732, sessionid=0x10009a83ed90000, setting cluster-up flag (Was=false) 2024-12-09T08:19:12,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10009a83ed90003, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:12,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:12,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33313-0x10009a83ed90001, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:12,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34083-0x10009a83ed90002, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:12,542 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T08:19:12,549 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5bda946059d1,41647,1733732350732 2024-12-09T08:19:12,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10009a83ed90003, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:12,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:12,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34083-0x10009a83ed90002, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:12,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33313-0x10009a83ed90001, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:12,605 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T08:19:12,610 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5bda946059d1,41647,1733732350732 2024-12-09T08:19:12,620 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-09T08:19:12,648 INFO [RS:0;5bda946059d1:33313 {}] regionserver.HRegionServer(746): ClusterId : 21dc23ac-0dff-402b-b794-53b2dfbe03d4 2024-12-09T08:19:12,648 INFO [RS:2;5bda946059d1:39779 {}] regionserver.HRegionServer(746): ClusterId : 21dc23ac-0dff-402b-b794-53b2dfbe03d4 2024-12-09T08:19:12,648 INFO [RS:1;5bda946059d1:34083 {}] regionserver.HRegionServer(746): ClusterId : 21dc23ac-0dff-402b-b794-53b2dfbe03d4 2024-12-09T08:19:12,650 DEBUG [RS:2;5bda946059d1:39779 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T08:19:12,650 DEBUG [RS:0;5bda946059d1:33313 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T08:19:12,650 DEBUG [RS:1;5bda946059d1:34083 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T08:19:12,660 DEBUG [RS:0;5bda946059d1:33313 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T08:19:12,660 DEBUG [RS:1;5bda946059d1:34083 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T08:19:12,660 DEBUG [RS:2;5bda946059d1:39779 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T08:19:12,660 DEBUG [RS:0;5bda946059d1:33313 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T08:19:12,660 DEBUG [RS:1;5bda946059d1:34083 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T08:19:12,660 DEBUG [RS:2;5bda946059d1:39779 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T08:19:12,669 DEBUG [RS:0;5bda946059d1:33313 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T08:19:12,669 DEBUG [RS:1;5bda946059d1:34083 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T08:19:12,669 DEBUG [RS:2;5bda946059d1:39779 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T08:19:12,670 DEBUG [RS:0;5bda946059d1:33313 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43c363b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5bda946059d1/172.17.0.2:0 2024-12-09T08:19:12,670 DEBUG [RS:2;5bda946059d1:39779 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@464a8cec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5bda946059d1/172.17.0.2:0 2024-12-09T08:19:12,670 DEBUG [RS:1;5bda946059d1:34083 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cf39f88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5bda946059d1/172.17.0.2:0 2024-12-09T08:19:12,688 DEBUG [RS:2;5bda946059d1:39779 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;5bda946059d1:39779 2024-12-09T08:19:12,688 DEBUG [RS:0;5bda946059d1:33313 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;5bda946059d1:33313 2024-12-09T08:19:12,688 DEBUG [RS:1;5bda946059d1:34083 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;5bda946059d1:34083 2024-12-09T08:19:12,691 INFO [RS:2;5bda946059d1:39779 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T08:19:12,691 INFO [RS:1;5bda946059d1:34083 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T08:19:12,691 INFO [RS:0;5bda946059d1:33313 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T08:19:12,691 INFO [RS:2;5bda946059d1:39779 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T08:19:12,691 INFO [RS:1;5bda946059d1:34083 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T08:19:12,691 INFO [RS:0;5bda946059d1:33313 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T08:19:12,691 DEBUG [RS:2;5bda946059d1:39779 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T08:19:12,691 DEBUG [RS:1;5bda946059d1:34083 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T08:19:12,691 DEBUG [RS:0;5bda946059d1:33313 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T08:19:12,693 INFO [RS:0;5bda946059d1:33313 {}] regionserver.HRegionServer(2659): reportForDuty to master=5bda946059d1,41647,1733732350732 with port=33313, startcode=1733732351482 2024-12-09T08:19:12,693 INFO [RS:2;5bda946059d1:39779 {}] regionserver.HRegionServer(2659): reportForDuty to master=5bda946059d1,41647,1733732350732 with port=39779, startcode=1733732351619 2024-12-09T08:19:12,693 INFO [RS:1;5bda946059d1:34083 {}] regionserver.HRegionServer(2659): reportForDuty to master=5bda946059d1,41647,1733732350732 with port=34083, startcode=1733732351576 2024-12-09T08:19:12,693 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T08:19:12,703 INFO [master/5bda946059d1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-09T08:19:12,704 DEBUG [RS:0;5bda946059d1:33313 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T08:19:12,704 DEBUG [RS:2;5bda946059d1:39779 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T08:19:12,704 DEBUG [RS:1;5bda946059d1:34083 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T08:19:12,709 INFO [master/5bda946059d1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T08:19:12,715 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 5bda946059d1,41647,1733732350732 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T08:19:12,722 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/5bda946059d1:0, corePoolSize=5, maxPoolSize=5 2024-12-09T08:19:12,722 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/5bda946059d1:0, corePoolSize=5, maxPoolSize=5 2024-12-09T08:19:12,723 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/5bda946059d1:0, corePoolSize=5, maxPoolSize=5 2024-12-09T08:19:12,723 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/5bda946059d1:0, corePoolSize=5, maxPoolSize=5 2024-12-09T08:19:12,723 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/5bda946059d1:0, corePoolSize=10, maxPoolSize=10 2024-12-09T08:19:12,723 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:12,723 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/5bda946059d1:0, corePoolSize=2, maxPoolSize=2 2024-12-09T08:19:12,723 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:12,730 INFO [master/5bda946059d1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733732382730 2024-12-09T08:19:12,731 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T08:19:12,731 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-09T08:19:12,732 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T08:19:12,733 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T08:19:12,736 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44453, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T08:19:12,736 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49287, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T08:19:12,736 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40625, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T08:19:12,738 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T08:19:12,738 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T08:19:12,739 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T08:19:12,739 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T08:19:12,739 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:12,739 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T08:19:12,740 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:12,744 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T08:19:12,744 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41647 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T08:19:12,745 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T08:19:12,746 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T08:19:12,748 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T08:19:12,748 INFO [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T08:19:12,749 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/5bda946059d1:0:becomeActiveMaster-HFileCleaner.large.0-1733732352749,5,FailOnTimeoutGroup] 2024-12-09T08:19:12,749 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41647 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T08:19:12,750 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/5bda946059d1:0:becomeActiveMaster-HFileCleaner.small.0-1733732352750,5,FailOnTimeoutGroup] 2024-12-09T08:19:12,750 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:12,751 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T08:19:12,751 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41647 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-09T08:19:12,778 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:12,778 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:12,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741831_1007 (size=1321) 2024-12-09T08:19:12,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741831_1007 (size=1321) 2024-12-09T08:19:12,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741831_1007 (size=1321) 2024-12-09T08:19:12,785 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-09T08:19:12,786 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3 2024-12-09T08:19:12,792 DEBUG [RS:0;5bda946059d1:33313 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-09T08:19:12,792 DEBUG [RS:1;5bda946059d1:34083 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-09T08:19:12,792 DEBUG [RS:2;5bda946059d1:39779 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-09T08:19:12,792 WARN [RS:0;5bda946059d1:33313 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-09T08:19:12,792 WARN [RS:2;5bda946059d1:39779 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-09T08:19:12,792 WARN [RS:1;5bda946059d1:34083 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-09T08:19:12,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741832_1008 (size=32) 2024-12-09T08:19:12,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741832_1008 (size=32) 2024-12-09T08:19:12,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741832_1008 (size=32) 2024-12-09T08:19:12,800 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:12,802 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T08:19:12,804 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T08:19:12,804 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:12,805 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:19:12,805 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T08:19:12,807 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T08:19:12,808 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:12,809 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:19:12,809 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T08:19:12,812 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T08:19:12,813 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:12,814 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:19:12,814 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T08:19:12,817 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T08:19:12,817 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:12,818 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:19:12,819 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T08:19:12,821 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/hbase/meta/1588230740 2024-12-09T08:19:12,821 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/hbase/meta/1588230740 2024-12-09T08:19:12,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T08:19:12,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T08:19:12,825 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T08:19:12,828 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T08:19:12,833 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:19:12,834 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74009148, jitterRate=0.10282224416732788}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T08:19:12,837 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733732352800Initializing all the Stores at 1733732352801 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733732352801Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733732352801Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732352801Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733732352801Cleaning up temporary data from old regions at 1733732352824 (+23 ms)Region opened successfully at 1733732352837 (+13 ms) 2024-12-09T08:19:12,837 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T08:19:12,837 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T08:19:12,838 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T08:19:12,838 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T08:19:12,838 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T08:19:12,839 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T08:19:12,840 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733732352837Disabling compacts and flushes for region at 1733732352837Disabling writes for close at 1733732352838 (+1 ms)Writing region close event to WAL at 1733732352839 (+1 ms)Closed at 1733732352839 2024-12-09T08:19:12,843 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T08:19:12,844 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-09T08:19:12,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T08:19:12,862 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T08:19:12,866 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T08:19:12,894 INFO [RS:1;5bda946059d1:34083 {}] regionserver.HRegionServer(2659): reportForDuty to master=5bda946059d1,41647,1733732350732 with port=34083, startcode=1733732351576 2024-12-09T08:19:12,894 INFO [RS:0;5bda946059d1:33313 {}] regionserver.HRegionServer(2659): reportForDuty to master=5bda946059d1,41647,1733732350732 with port=33313, startcode=1733732351482 2024-12-09T08:19:12,894 INFO [RS:2;5bda946059d1:39779 {}] regionserver.HRegionServer(2659): reportForDuty to master=5bda946059d1,41647,1733732350732 with port=39779, startcode=1733732351619 2024-12-09T08:19:12,895 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41647 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5bda946059d1,33313,1733732351482 2024-12-09T08:19:12,898 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41647 {}] master.ServerManager(517): Registering regionserver=5bda946059d1,33313,1733732351482 2024-12-09T08:19:12,907 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41647 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5bda946059d1,39779,1733732351619 2024-12-09T08:19:12,907 DEBUG [RS:0;5bda946059d1:33313 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3 2024-12-09T08:19:12,907 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41647 {}] master.ServerManager(517): Registering regionserver=5bda946059d1,39779,1733732351619 2024-12-09T08:19:12,907 DEBUG [RS:0;5bda946059d1:33313 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45647 2024-12-09T08:19:12,907 DEBUG [RS:0;5bda946059d1:33313 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T08:19:12,912 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41647 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5bda946059d1,34083,1733732351576 2024-12-09T08:19:12,912 DEBUG [RS:2;5bda946059d1:39779 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3 2024-12-09T08:19:12,912 DEBUG [RS:2;5bda946059d1:39779 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45647 2024-12-09T08:19:12,912 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41647 {}] master.ServerManager(517): Registering regionserver=5bda946059d1,34083,1733732351576 2024-12-09T08:19:12,912 DEBUG [RS:2;5bda946059d1:39779 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T08:19:12,915 DEBUG [RS:1;5bda946059d1:34083 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3 2024-12-09T08:19:12,915 DEBUG [RS:1;5bda946059d1:34083 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45647 2024-12-09T08:19:12,915 DEBUG [RS:1;5bda946059d1:34083 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T08:19:12,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T08:19:12,959 DEBUG [RS:0;5bda946059d1:33313 {}] zookeeper.ZKUtil(111): regionserver:33313-0x10009a83ed90001, quorum=127.0.0.1:55499, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/5bda946059d1,33313,1733732351482 2024-12-09T08:19:12,959 WARN [RS:0;5bda946059d1:33313 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T08:19:12,959 DEBUG [RS:2;5bda946059d1:39779 {}] zookeeper.ZKUtil(111): regionserver:39779-0x10009a83ed90003, quorum=127.0.0.1:55499, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/5bda946059d1,39779,1733732351619 2024-12-09T08:19:12,959 DEBUG [RS:1;5bda946059d1:34083 {}] zookeeper.ZKUtil(111): regionserver:34083-0x10009a83ed90002, quorum=127.0.0.1:55499, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/5bda946059d1,34083,1733732351576 2024-12-09T08:19:12,959 WARN [RS:2;5bda946059d1:39779 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T08:19:12,959 INFO [RS:0;5bda946059d1:33313 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:12,959 WARN [RS:1;5bda946059d1:34083 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T08:19:12,959 INFO [RS:2;5bda946059d1:39779 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:12,960 INFO [RS:1;5bda946059d1:34083 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:12,960 DEBUG [RS:2;5bda946059d1:39779 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,39779,1733732351619 2024-12-09T08:19:12,960 DEBUG [RS:0;5bda946059d1:33313 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,33313,1733732351482 2024-12-09T08:19:12,960 DEBUG [RS:1;5bda946059d1:34083 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,34083,1733732351576 2024-12-09T08:19:12,961 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5bda946059d1,34083,1733732351576] 2024-12-09T08:19:12,962 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5bda946059d1,39779,1733732351619] 2024-12-09T08:19:12,962 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5bda946059d1,33313,1733732351482] 2024-12-09T08:19:12,986 INFO [RS:1;5bda946059d1:34083 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T08:19:12,986 INFO [RS:0;5bda946059d1:33313 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T08:19:12,986 INFO [RS:2;5bda946059d1:39779 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T08:19:13,001 INFO [RS:2;5bda946059d1:39779 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T08:19:13,001 INFO [RS:0;5bda946059d1:33313 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T08:19:13,001 INFO [RS:1;5bda946059d1:34083 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T08:19:13,006 INFO [RS:2;5bda946059d1:39779 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T08:19:13,006 INFO [RS:1;5bda946059d1:34083 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T08:19:13,006 INFO [RS:0;5bda946059d1:33313 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T08:19:13,006 INFO [RS:2;5bda946059d1:39779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,006 INFO [RS:0;5bda946059d1:33313 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,006 INFO [RS:1;5bda946059d1:34083 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,007 INFO [RS:0;5bda946059d1:33313 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T08:19:13,007 INFO [RS:2;5bda946059d1:39779 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T08:19:13,009 INFO [RS:1;5bda946059d1:34083 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T08:19:13,012 INFO [RS:2;5bda946059d1:39779 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T08:19:13,012 INFO [RS:0;5bda946059d1:33313 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T08:19:13,012 INFO [RS:1;5bda946059d1:34083 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T08:19:13,014 INFO [RS:0;5bda946059d1:33313 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,014 INFO [RS:2;5bda946059d1:39779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,014 INFO [RS:1;5bda946059d1:34083 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,014 DEBUG [RS:2;5bda946059d1:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,014 DEBUG [RS:0;5bda946059d1:33313 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,014 DEBUG [RS:1;5bda946059d1:34083 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,014 DEBUG [RS:2;5bda946059d1:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,014 DEBUG [RS:0;5bda946059d1:33313 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,014 DEBUG [RS:1;5bda946059d1:34083 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,014 DEBUG [RS:2;5bda946059d1:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,014 DEBUG [RS:0;5bda946059d1:33313 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,014 DEBUG [RS:1;5bda946059d1:34083 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,014 DEBUG [RS:2;5bda946059d1:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,014 DEBUG [RS:0;5bda946059d1:33313 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,014 DEBUG [RS:2;5bda946059d1:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,014 DEBUG [RS:0;5bda946059d1:33313 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,014 DEBUG [RS:1;5bda946059d1:34083 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,015 DEBUG [RS:2;5bda946059d1:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0, corePoolSize=2, maxPoolSize=2 2024-12-09T08:19:13,015 DEBUG [RS:0;5bda946059d1:33313 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0, corePoolSize=2, maxPoolSize=2 2024-12-09T08:19:13,015 DEBUG [RS:1;5bda946059d1:34083 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,015 DEBUG [RS:2;5bda946059d1:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,015 DEBUG [RS:0;5bda946059d1:33313 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,015 DEBUG [RS:1;5bda946059d1:34083 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0, corePoolSize=2, maxPoolSize=2 2024-12-09T08:19:13,015 DEBUG [RS:2;5bda946059d1:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,015 DEBUG [RS:0;5bda946059d1:33313 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,015 DEBUG [RS:0;5bda946059d1:33313 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,015 DEBUG [RS:2;5bda946059d1:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,015 DEBUG [RS:1;5bda946059d1:34083 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,015 DEBUG [RS:0;5bda946059d1:33313 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,015 DEBUG [RS:2;5bda946059d1:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,015 DEBUG [RS:1;5bda946059d1:34083 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,015 DEBUG [RS:0;5bda946059d1:33313 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,015 DEBUG [RS:2;5bda946059d1:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,015 DEBUG [RS:0;5bda946059d1:33313 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,015 DEBUG [RS:2;5bda946059d1:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,015 DEBUG [RS:1;5bda946059d1:34083 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,015 DEBUG [RS:0;5bda946059d1:33313 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5bda946059d1:0, corePoolSize=3, maxPoolSize=3 2024-12-09T08:19:13,015 DEBUG [RS:2;5bda946059d1:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5bda946059d1:0, corePoolSize=3, maxPoolSize=3 2024-12-09T08:19:13,015 DEBUG [RS:1;5bda946059d1:34083 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,015 DEBUG [RS:0;5bda946059d1:33313 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0, corePoolSize=3, maxPoolSize=3 2024-12-09T08:19:13,015 DEBUG [RS:2;5bda946059d1:39779 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0, corePoolSize=3, maxPoolSize=3 2024-12-09T08:19:13,015 DEBUG [RS:1;5bda946059d1:34083 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,016 DEBUG [RS:1;5bda946059d1:34083 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5bda946059d1:0, corePoolSize=1, maxPoolSize=1 2024-12-09T08:19:13,016 DEBUG [RS:1;5bda946059d1:34083 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5bda946059d1:0, corePoolSize=3, maxPoolSize=3 2024-12-09T08:19:13,016 DEBUG [RS:1;5bda946059d1:34083 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5bda946059d1:0, corePoolSize=3, maxPoolSize=3 2024-12-09T08:19:13,017 WARN [5bda946059d1:41647 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T08:19:13,017 INFO [RS:1;5bda946059d1:34083 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,017 INFO [RS:2;5bda946059d1:39779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,018 INFO [RS:1;5bda946059d1:34083 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,018 INFO [RS:2;5bda946059d1:39779 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,018 INFO [RS:0;5bda946059d1:33313 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,018 INFO [RS:1;5bda946059d1:34083 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,018 INFO [RS:2;5bda946059d1:39779 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,018 INFO [RS:0;5bda946059d1:33313 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,018 INFO [RS:1;5bda946059d1:34083 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,018 INFO [RS:2;5bda946059d1:39779 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,018 INFO [RS:0;5bda946059d1:33313 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,018 INFO [RS:1;5bda946059d1:34083 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,018 INFO [RS:2;5bda946059d1:39779 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,018 INFO [RS:0;5bda946059d1:33313 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,018 INFO [RS:1;5bda946059d1:34083 {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,34083,1733732351576-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T08:19:13,018 INFO [RS:2;5bda946059d1:39779 {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,39779,1733732351619-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T08:19:13,018 INFO [RS:0;5bda946059d1:33313 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,018 INFO [RS:0;5bda946059d1:33313 {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,33313,1733732351482-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T08:19:13,039 INFO [RS:0;5bda946059d1:33313 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T08:19:13,039 INFO [RS:1;5bda946059d1:34083 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T08:19:13,039 INFO [RS:2;5bda946059d1:39779 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T08:19:13,041 INFO [RS:0;5bda946059d1:33313 {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,33313,1733732351482-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,041 INFO [RS:2;5bda946059d1:39779 {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,39779,1733732351619-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,041 INFO [RS:1;5bda946059d1:34083 {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,34083,1733732351576-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,041 INFO [RS:0;5bda946059d1:33313 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,041 INFO [RS:2;5bda946059d1:39779 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,041 INFO [RS:1;5bda946059d1:34083 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,041 INFO [RS:1;5bda946059d1:34083 {}] regionserver.Replication(171): 5bda946059d1,34083,1733732351576 started 2024-12-09T08:19:13,041 INFO [RS:2;5bda946059d1:39779 {}] regionserver.Replication(171): 5bda946059d1,39779,1733732351619 started 2024-12-09T08:19:13,041 INFO [RS:0;5bda946059d1:33313 {}] regionserver.Replication(171): 5bda946059d1,33313,1733732351482 started 2024-12-09T08:19:13,058 INFO [RS:0;5bda946059d1:33313 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,059 INFO [RS:0;5bda946059d1:33313 {}] regionserver.HRegionServer(1482): Serving as 5bda946059d1,33313,1733732351482, RpcServer on 5bda946059d1/172.17.0.2:33313, sessionid=0x10009a83ed90001 2024-12-09T08:19:13,059 DEBUG [RS:0;5bda946059d1:33313 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T08:19:13,059 DEBUG [RS:0;5bda946059d1:33313 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5bda946059d1,33313,1733732351482 2024-12-09T08:19:13,060 DEBUG [RS:0;5bda946059d1:33313 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5bda946059d1,33313,1733732351482' 2024-12-09T08:19:13,060 DEBUG [RS:0;5bda946059d1:33313 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T08:19:13,060 DEBUG [RS:0;5bda946059d1:33313 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T08:19:13,061 DEBUG [RS:0;5bda946059d1:33313 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T08:19:13,061 DEBUG [RS:0;5bda946059d1:33313 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T08:19:13,061 DEBUG [RS:0;5bda946059d1:33313 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5bda946059d1,33313,1733732351482 2024-12-09T08:19:13,061 DEBUG [RS:0;5bda946059d1:33313 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5bda946059d1,33313,1733732351482' 2024-12-09T08:19:13,061 DEBUG [RS:0;5bda946059d1:33313 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T08:19:13,062 DEBUG [RS:0;5bda946059d1:33313 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T08:19:13,062 DEBUG [RS:0;5bda946059d1:33313 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T08:19:13,062 INFO [RS:0;5bda946059d1:33313 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T08:19:13,062 INFO [RS:0;5bda946059d1:33313 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T08:19:13,063 INFO [RS:2;5bda946059d1:39779 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,063 INFO [RS:1;5bda946059d1:34083 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:13,063 INFO [RS:1;5bda946059d1:34083 {}] regionserver.HRegionServer(1482): Serving as 5bda946059d1,34083,1733732351576, RpcServer on 5bda946059d1/172.17.0.2:34083, sessionid=0x10009a83ed90002 2024-12-09T08:19:13,063 INFO [RS:2;5bda946059d1:39779 {}] regionserver.HRegionServer(1482): Serving as 5bda946059d1,39779,1733732351619, RpcServer on 5bda946059d1/172.17.0.2:39779, sessionid=0x10009a83ed90003 2024-12-09T08:19:13,063 DEBUG [RS:2;5bda946059d1:39779 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T08:19:13,063 DEBUG [RS:1;5bda946059d1:34083 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T08:19:13,063 DEBUG [RS:2;5bda946059d1:39779 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5bda946059d1,39779,1733732351619 2024-12-09T08:19:13,063 DEBUG [RS:1;5bda946059d1:34083 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5bda946059d1,34083,1733732351576 2024-12-09T08:19:13,064 DEBUG [RS:2;5bda946059d1:39779 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5bda946059d1,39779,1733732351619' 2024-12-09T08:19:13,064 DEBUG [RS:1;5bda946059d1:34083 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5bda946059d1,34083,1733732351576' 2024-12-09T08:19:13,064 DEBUG [RS:1;5bda946059d1:34083 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T08:19:13,064 DEBUG [RS:2;5bda946059d1:39779 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T08:19:13,064 DEBUG [RS:2;5bda946059d1:39779 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T08:19:13,064 DEBUG [RS:1;5bda946059d1:34083 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T08:19:13,065 DEBUG [RS:1;5bda946059d1:34083 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T08:19:13,065 DEBUG [RS:2;5bda946059d1:39779 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T08:19:13,065 DEBUG [RS:1;5bda946059d1:34083 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T08:19:13,065 DEBUG [RS:1;5bda946059d1:34083 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5bda946059d1,34083,1733732351576 2024-12-09T08:19:13,065 DEBUG [RS:2;5bda946059d1:39779 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T08:19:13,065 DEBUG [RS:1;5bda946059d1:34083 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5bda946059d1,34083,1733732351576' 2024-12-09T08:19:13,065 DEBUG [RS:2;5bda946059d1:39779 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5bda946059d1,39779,1733732351619 2024-12-09T08:19:13,065 DEBUG [RS:1;5bda946059d1:34083 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T08:19:13,065 DEBUG [RS:2;5bda946059d1:39779 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5bda946059d1,39779,1733732351619' 2024-12-09T08:19:13,065 DEBUG [RS:2;5bda946059d1:39779 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T08:19:13,066 DEBUG [RS:1;5bda946059d1:34083 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T08:19:13,066 DEBUG [RS:2;5bda946059d1:39779 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T08:19:13,066 DEBUG [RS:1;5bda946059d1:34083 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T08:19:13,066 DEBUG [RS:2;5bda946059d1:39779 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T08:19:13,066 INFO [RS:1;5bda946059d1:34083 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T08:19:13,066 INFO [RS:2;5bda946059d1:39779 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T08:19:13,066 INFO [RS:1;5bda946059d1:34083 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T08:19:13,066 INFO [RS:2;5bda946059d1:39779 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T08:19:13,171 INFO [RS:0;5bda946059d1:33313 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:19:13,171 INFO [RS:1;5bda946059d1:34083 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:19:13,172 INFO [RS:2;5bda946059d1:39779 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:19:13,176 INFO [RS:0;5bda946059d1:33313 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5bda946059d1%2C33313%2C1733732351482, suffix=, logDir=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,33313,1733732351482, archiveDir=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/oldWALs, maxLogs=32 2024-12-09T08:19:13,176 INFO [RS:1;5bda946059d1:34083 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5bda946059d1%2C34083%2C1733732351576, suffix=, logDir=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,34083,1733732351576, archiveDir=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/oldWALs, maxLogs=32 2024-12-09T08:19:13,176 INFO [RS:2;5bda946059d1:39779 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5bda946059d1%2C39779%2C1733732351619, suffix=, logDir=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,39779,1733732351619, archiveDir=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/oldWALs, maxLogs=32 2024-12-09T08:19:13,190 DEBUG [RS:0;5bda946059d1:33313 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,33313,1733732351482/5bda946059d1%2C33313%2C1733732351482.1733732353179, exclude list is [], retry=0 2024-12-09T08:19:13,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:13,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:13,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:13,197 DEBUG [RS:1;5bda946059d1:34083 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,34083,1733732351576/5bda946059d1%2C34083%2C1733732351576.1733732353180, exclude list is [], retry=0 2024-12-09T08:19:13,197 DEBUG [RS:2;5bda946059d1:39779 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,39779,1733732351619/5bda946059d1%2C39779%2C1733732351619.1733732353180, exclude list is [], retry=0 2024-12-09T08:19:13,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:13,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:13,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:13,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:13,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:13,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:13,232 INFO [RS:0;5bda946059d1:33313 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,33313,1733732351482/5bda946059d1%2C33313%2C1733732351482.1733732353179 2024-12-09T08:19:13,233 DEBUG [RS:0;5bda946059d1:33313 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42925:42925),(127.0.0.1/127.0.0.1:41283:41283),(127.0.0.1/127.0.0.1:42683:42683)] 2024-12-09T08:19:13,237 INFO [RS:1;5bda946059d1:34083 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,34083,1733732351576/5bda946059d1%2C34083%2C1733732351576.1733732353180 2024-12-09T08:19:13,239 DEBUG [RS:1;5bda946059d1:34083 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42925:42925),(127.0.0.1/127.0.0.1:42683:42683),(127.0.0.1/127.0.0.1:41283:41283)] 2024-12-09T08:19:13,243 INFO [RS:2;5bda946059d1:39779 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,39779,1733732351619/5bda946059d1%2C39779%2C1733732351619.1733732353180 2024-12-09T08:19:13,244 DEBUG [RS:2;5bda946059d1:39779 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41283:41283),(127.0.0.1/127.0.0.1:42683:42683),(127.0.0.1/127.0.0.1:42925:42925)] 2024-12-09T08:19:13,269 DEBUG [5bda946059d1:41647 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-09T08:19:13,275 DEBUG [5bda946059d1:41647 {}] balancer.BalancerClusterState(204): Hosts are {5bda946059d1=0} racks are {/default-rack=0} 2024-12-09T08:19:13,282 DEBUG [5bda946059d1:41647 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T08:19:13,282 DEBUG [5bda946059d1:41647 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T08:19:13,282 DEBUG [5bda946059d1:41647 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-09T08:19:13,282 DEBUG [5bda946059d1:41647 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T08:19:13,282 DEBUG [5bda946059d1:41647 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T08:19:13,282 DEBUG [5bda946059d1:41647 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-09T08:19:13,282 INFO [5bda946059d1:41647 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T08:19:13,282 INFO [5bda946059d1:41647 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T08:19:13,282 INFO [5bda946059d1:41647 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-09T08:19:13,282 DEBUG [5bda946059d1:41647 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T08:19:13,288 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=5bda946059d1,39779,1733732351619 2024-12-09T08:19:13,293 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5bda946059d1,39779,1733732351619, state=OPENING 2024-12-09T08:19:13,321 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T08:19:13,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34083-0x10009a83ed90002, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:13,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10009a83ed90003, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:13,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:13,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33313-0x10009a83ed90001, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:13,333 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T08:19:13,333 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T08:19:13,333 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T08:19:13,333 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T08:19:13,335 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T08:19:13,338 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=5bda946059d1,39779,1733732351619}] 2024-12-09T08:19:13,518 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T08:19:13,519 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43945, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T08:19:13,531 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T08:19:13,532 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:13,532 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T08:19:13,535 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5bda946059d1%2C39779%2C1733732351619.meta, suffix=.meta, logDir=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,39779,1733732351619, archiveDir=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/oldWALs, maxLogs=32 2024-12-09T08:19:13,551 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,39779,1733732351619/5bda946059d1%2C39779%2C1733732351619.meta.1733732353537.meta, exclude list is [], retry=0 2024-12-09T08:19:13,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:13,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:13,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:13,583 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,39779,1733732351619/5bda946059d1%2C39779%2C1733732351619.meta.1733732353537.meta 2024-12-09T08:19:13,583 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42683:42683),(127.0.0.1/127.0.0.1:41283:41283),(127.0.0.1/127.0.0.1:42925:42925)] 2024-12-09T08:19:13,584 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:19:13,585 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T08:19:13,587 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T08:19:13,591 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T08:19:13,595 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T08:19:13,595 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:13,596 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T08:19:13,596 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T08:19:13,599 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T08:19:13,600 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T08:19:13,600 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:13,601 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:19:13,601 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T08:19:13,602 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T08:19:13,603 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:13,603 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:19:13,604 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T08:19:13,605 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T08:19:13,605 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:13,606 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:19:13,606 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T08:19:13,607 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T08:19:13,607 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:13,608 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T08:19:13,608 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T08:19:13,609 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/hbase/meta/1588230740 2024-12-09T08:19:13,612 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/hbase/meta/1588230740 2024-12-09T08:19:13,614 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T08:19:13,614 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T08:19:13,615 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T08:19:13,617 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T08:19:13,619 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66110917, jitterRate=-0.014870569109916687}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T08:19:13,619 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T08:19:13,620 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733732353596Writing region info on filesystem at 1733732353596Initializing all the Stores at 1733732353598 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733732353598Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733732353598Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732353599 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733732353599Cleaning up temporary data from old regions at 1733732353614 (+15 ms)Running coprocessor post-open hooks at 1733732353619 (+5 ms)Region opened successfully at 1733732353620 (+1 ms) 2024-12-09T08:19:13,627 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733732353509 2024-12-09T08:19:13,637 DEBUG [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T08:19:13,637 INFO [RS_OPEN_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T08:19:13,639 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=5bda946059d1,39779,1733732351619 2024-12-09T08:19:13,641 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5bda946059d1,39779,1733732351619, state=OPEN 2024-12-09T08:19:13,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33313-0x10009a83ed90001, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T08:19:13,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10009a83ed90003, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T08:19:13,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T08:19:13,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34083-0x10009a83ed90002, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T08:19:13,898 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T08:19:13,898 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T08:19:13,898 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T08:19:13,898 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T08:19:13,899 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=5bda946059d1,39779,1733732351619 2024-12-09T08:19:13,906 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T08:19:13,907 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5bda946059d1,39779,1733732351619 in 562 msec 2024-12-09T08:19:13,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T08:19:13,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.0570 sec 2024-12-09T08:19:13,914 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T08:19:13,914 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-09T08:19:13,932 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:19:13,933 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,39779,1733732351619, seqNum=-1] 2024-12-09T08:19:13,950 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:19:13,952 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35753, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:19:13,990 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3240 sec 2024-12-09T08:19:13,991 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733732353991, completionTime=-1 2024-12-09T08:19:13,994 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-09T08:19:13,996 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T08:19:14,031 INFO [master/5bda946059d1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-09T08:19:14,031 INFO [master/5bda946059d1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733732414031 2024-12-09T08:19:14,031 INFO [master/5bda946059d1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733732474031 2024-12-09T08:19:14,031 INFO [master/5bda946059d1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 35 msec 2024-12-09T08:19:14,032 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-09T08:19:14,039 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,41647,1733732350732-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:14,040 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,41647,1733732350732-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:14,040 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,41647,1733732350732-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:14,042 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-5bda946059d1:41647, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:14,042 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:14,043 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:14,048 DEBUG [master/5bda946059d1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-09T08:19:14,067 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.355sec 2024-12-09T08:19:14,068 INFO [master/5bda946059d1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T08:19:14,069 INFO [master/5bda946059d1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T08:19:14,070 INFO [master/5bda946059d1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T08:19:14,070 INFO [master/5bda946059d1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T08:19:14,070 INFO [master/5bda946059d1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T08:19:14,071 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,41647,1733732350732-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T08:19:14,071 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,41647,1733732350732-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T08:19:14,076 DEBUG [master/5bda946059d1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-09T08:19:14,077 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T08:19:14,077 INFO [master/5bda946059d1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5bda946059d1,41647,1733732350732-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:14,159 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16a5f0dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:19:14,160 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5bda946059d1,41647,-1 for getting cluster id 2024-12-09T08:19:14,163 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T08:19:14,171 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '21dc23ac-0dff-402b-b794-53b2dfbe03d4' 2024-12-09T08:19:14,173 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T08:19:14,173 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "21dc23ac-0dff-402b-b794-53b2dfbe03d4" 2024-12-09T08:19:14,174 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4792eea0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:19:14,174 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5bda946059d1,41647,-1] 2024-12-09T08:19:14,177 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T08:19:14,178 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:19:14,179 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T08:19:14,182 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f754b13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T08:19:14,182 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T08:19:14,188 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5bda946059d1,39779,1733732351619, seqNum=-1] 2024-12-09T08:19:14,188 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:19:14,190 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:19:14,208 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=5bda946059d1,41647,1733732350732 2024-12-09T08:19:14,208 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:45647/hbase 2024-12-09T08:19:14,220 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=361, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=271, ProcessCount=11, AvailableMemoryMB=4790 2024-12-09T08:19:14,236 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:14,239 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:14,240 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:19:14,245 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-79624354, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/hregion-79624354, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:14,259 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-79624354/hregion-79624354.1733732354246, exclude list is [], retry=0 2024-12-09T08:19:14,263 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:14,263 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:14,264 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:14,289 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-79624354/hregion-79624354.1733732354246 2024-12-09T08:19:14,289 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42925:42925),(127.0.0.1/127.0.0.1:42683:42683),(127.0.0.1/127.0.0.1:41283:41283)] 2024-12-09T08:19:14,289 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d0d86132b03ea6f9edcee6d7811b1f14, NAME => 'testReplayEditsWrittenIntoWAL,,1733732354237.d0d86132b03ea6f9edcee6d7811b1f14.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45647/hbase 2024-12-09T08:19:14,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741838_1014 (size=64) 2024-12-09T08:19:14,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741838_1014 (size=64) 2024-12-09T08:19:14,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741838_1014 (size=64) 2024-12-09T08:19:14,302 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733732354237.d0d86132b03ea6f9edcee6d7811b1f14.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:14,305 INFO [StoreOpener-d0d86132b03ea6f9edcee6d7811b1f14-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d0d86132b03ea6f9edcee6d7811b1f14 2024-12-09T08:19:14,308 INFO [StoreOpener-d0d86132b03ea6f9edcee6d7811b1f14-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0d86132b03ea6f9edcee6d7811b1f14 columnFamilyName a 2024-12-09T08:19:14,308 DEBUG [StoreOpener-d0d86132b03ea6f9edcee6d7811b1f14-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:14,309 INFO [StoreOpener-d0d86132b03ea6f9edcee6d7811b1f14-1 {}] regionserver.HStore(327): Store=d0d86132b03ea6f9edcee6d7811b1f14/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:14,309 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d0d86132b03ea6f9edcee6d7811b1f14 2024-12-09T08:19:14,310 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14 2024-12-09T08:19:14,311 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14 2024-12-09T08:19:14,311 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d0d86132b03ea6f9edcee6d7811b1f14 2024-12-09T08:19:14,311 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d0d86132b03ea6f9edcee6d7811b1f14 2024-12-09T08:19:14,314 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d0d86132b03ea6f9edcee6d7811b1f14 2024-12-09T08:19:14,317 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:19:14,318 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d0d86132b03ea6f9edcee6d7811b1f14; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66620655, jitterRate=-0.00727488100528717}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T08:19:14,319 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d0d86132b03ea6f9edcee6d7811b1f14: Writing region info on filesystem at 1733732354303Initializing all the Stores at 1733732354305 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732354305Cleaning up temporary data from old regions at 1733732354311 (+6 ms)Region opened successfully at 1733732354319 (+8 ms) 2024-12-09T08:19:14,319 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d0d86132b03ea6f9edcee6d7811b1f14, disabling compactions & flushes 2024-12-09T08:19:14,319 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733732354237.d0d86132b03ea6f9edcee6d7811b1f14. 2024-12-09T08:19:14,319 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733732354237.d0d86132b03ea6f9edcee6d7811b1f14. 2024-12-09T08:19:14,319 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733732354237.d0d86132b03ea6f9edcee6d7811b1f14. after waiting 0 ms 2024-12-09T08:19:14,319 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733732354237.d0d86132b03ea6f9edcee6d7811b1f14. 2024-12-09T08:19:14,320 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733732354237.d0d86132b03ea6f9edcee6d7811b1f14. 2024-12-09T08:19:14,320 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d0d86132b03ea6f9edcee6d7811b1f14: Waiting for close lock at 1733732354319Disabling compacts and flushes for region at 1733732354319Disabling writes for close at 1733732354319Writing region close event to WAL at 1733732354320 (+1 ms)Closed at 1733732354320 2024-12-09T08:19:14,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741837_1013 (size=175) 2024-12-09T08:19:14,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741837_1013 (size=175) 2024-12-09T08:19:14,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741837_1013 (size=175) 2024-12-09T08:19:14,331 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:19:14,331 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-79624354:(num 1733732354246) 2024-12-09T08:19:14,333 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T08:19:14,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741839_1015 (size=469) 2024-12-09T08:19:14,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741839_1015 (size=469) 2024-12-09T08:19:14,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741839_1015 (size=469) 2024-12-09T08:19:14,373 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T08:19:14,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741840_1016 (size=322) 2024-12-09T08:19:14,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741840_1016 (size=322) 2024-12-09T08:19:14,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741840_1016 (size=322) 2024-12-09T08:19:14,427 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235/wal-1, size=469 (469bytes) 2024-12-09T08:19:14,428 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-09T08:19:14,428 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-09T08:19:14,428 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235/wal-1 2024-12-09T08:19:14,433 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235/wal-1 after 3ms 2024-12-09T08:19:14,458 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:14,459 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235/wal-1 took 33ms 2024-12-09T08:19:14,463 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235/wal-1 so closing down 2024-12-09T08:19:14,464 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:19:14,467 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-09T08:19:14,491 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14/recovered.edits/0000000000000000001-wal-1.temp 2024-12-09T08:19:14,492 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:19:14,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741841_1017 (size=469) 2024-12-09T08:19:14,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741841_1017 (size=469) 2024-12-09T08:19:14,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741841_1017 (size=469) 2024-12-09T08:19:14,504 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-09T08:19:14,507 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14/recovered.edits/0000000000000000002 2024-12-09T08:19:14,512 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-09T08:19:14,513 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235/wal-1, journal: Splitting hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235/wal-1, size=469 (469bytes) at 1733732354427Finishing writing output for hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235/wal-1 so closing down at 1733732354464 (+37 ms)Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14/recovered.edits/0000000000000000001-wal-1.temp at 1733732354492 (+28 ms)3 split writer threads finished at 1733732354492Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733732354504 (+12 ms)Rename recovered edits hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14/recovered.edits/0000000000000000002 at 1733732354507 (+3 ms)Processed 2 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733732354513 (+6 ms) 2024-12-09T08:19:14,526 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235/wal-2, size=322 (322bytes) 2024-12-09T08:19:14,526 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235/wal-2 2024-12-09T08:19:14,527 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235/wal-2 after 1ms 2024-12-09T08:19:14,552 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:14,552 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235/wal-2 took 26ms 2024-12-09T08:19:14,554 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235/wal-2 so closing down 2024-12-09T08:19:14,554 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:19:14,557 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-09T08:19:14,581 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14/recovered.edits/0000000000000000002-wal-2.temp 2024-12-09T08:19:14,582 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:19:14,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741842_1018 (size=322) 2024-12-09T08:19:14,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741842_1018 (size=322) 2024-12-09T08:19:14,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741842_1018 (size=322) 2024-12-09T08:19:14,591 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-09T08:19:14,618 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:14,621 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-12-09T08:19:14,622 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 70 ms; skipped=0; WAL=hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-09T08:19:14,622 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235/wal-2, journal: Splitting hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235/wal-2, size=322 (322bytes) at 1733732354526Finishing writing output for hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235/wal-2 so closing down at 1733732354554 (+28 ms)Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14/recovered.edits/0000000000000000002-wal-2.temp at 1733732354581 (+27 ms)3 split writer threads finished at 1733732354582 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733732354591 (+9 ms)Processed 1 edits across 1 Regions in 70 ms; skipped=0; WAL=hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733732354622 (+31 ms) 2024-12-09T08:19:14,623 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:19:14,625 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:14,636 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235/wal.1733732354625, exclude list is [], retry=0 2024-12-09T08:19:14,640 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:14,641 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:14,641 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:14,665 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235/wal.1733732354625 2024-12-09T08:19:14,665 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42925:42925),(127.0.0.1/127.0.0.1:42683:42683),(127.0.0.1/127.0.0.1:41283:41283)] 2024-12-09T08:19:14,665 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d0d86132b03ea6f9edcee6d7811b1f14, NAME => 'testReplayEditsWrittenIntoWAL,,1733732354237.d0d86132b03ea6f9edcee6d7811b1f14.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:19:14,665 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733732354237.d0d86132b03ea6f9edcee6d7811b1f14.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:14,665 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d0d86132b03ea6f9edcee6d7811b1f14 2024-12-09T08:19:14,665 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d0d86132b03ea6f9edcee6d7811b1f14 2024-12-09T08:19:14,667 INFO [StoreOpener-d0d86132b03ea6f9edcee6d7811b1f14-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d0d86132b03ea6f9edcee6d7811b1f14 2024-12-09T08:19:14,669 INFO [StoreOpener-d0d86132b03ea6f9edcee6d7811b1f14-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0d86132b03ea6f9edcee6d7811b1f14 columnFamilyName a 2024-12-09T08:19:14,669 DEBUG [StoreOpener-d0d86132b03ea6f9edcee6d7811b1f14-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:14,670 INFO [StoreOpener-d0d86132b03ea6f9edcee6d7811b1f14-1 {}] regionserver.HStore(327): Store=d0d86132b03ea6f9edcee6d7811b1f14/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:14,670 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d0d86132b03ea6f9edcee6d7811b1f14 2024-12-09T08:19:14,671 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14 2024-12-09T08:19:14,674 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14 2024-12-09T08:19:14,675 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14/recovered.edits/0000000000000000002 2024-12-09T08:19:14,700 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:14,705 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14/recovered.edits/0000000000000000002 2024-12-09T08:19:14,708 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d0d86132b03ea6f9edcee6d7811b1f14 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-09T08:19:14,751 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14/.tmp/a/d5e3a618d2a24f01afca4e2c9e83dad8 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733732354332/Put/seqid=0 2024-12-09T08:19:14,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741844_1020 (size=5170) 2024-12-09T08:19:14,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741844_1020 (size=5170) 2024-12-09T08:19:14,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741844_1020 (size=5170) 2024-12-09T08:19:14,762 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14/.tmp/a/d5e3a618d2a24f01afca4e2c9e83dad8 2024-12-09T08:19:14,799 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14/.tmp/a/d5e3a618d2a24f01afca4e2c9e83dad8 as hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14/a/d5e3a618d2a24f01afca4e2c9e83dad8 2024-12-09T08:19:14,808 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14/a/d5e3a618d2a24f01afca4e2c9e83dad8, entries=2, sequenceid=2, filesize=5.0 K 2024-12-09T08:19:14,815 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for d0d86132b03ea6f9edcee6d7811b1f14 in 103ms, sequenceid=2, compaction requested=false; wal=null 2024-12-09T08:19:14,817 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14/recovered.edits/0000000000000000002 2024-12-09T08:19:14,818 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d0d86132b03ea6f9edcee6d7811b1f14 2024-12-09T08:19:14,818 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d0d86132b03ea6f9edcee6d7811b1f14 2024-12-09T08:19:14,822 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d0d86132b03ea6f9edcee6d7811b1f14 2024-12-09T08:19:14,825 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/d0d86132b03ea6f9edcee6d7811b1f14/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-09T08:19:14,827 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d0d86132b03ea6f9edcee6d7811b1f14; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66358072, jitterRate=-0.01118767261505127}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T08:19:14,828 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d0d86132b03ea6f9edcee6d7811b1f14: Writing region info on filesystem at 1733732354666Initializing all the Stores at 1733732354667 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732354667Obtaining lock to block concurrent updates at 1733732354708 (+41 ms)Preparing flush snapshotting stores in d0d86132b03ea6f9edcee6d7811b1f14 at 1733732354708Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733732354237.d0d86132b03ea6f9edcee6d7811b1f14., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733732354710 (+2 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733732354237.d0d86132b03ea6f9edcee6d7811b1f14. at 1733732354710Flushing d0d86132b03ea6f9edcee6d7811b1f14/a: creating writer at 1733732354711 (+1 ms)Flushing d0d86132b03ea6f9edcee6d7811b1f14/a: appending metadata at 1733732354743 (+32 ms)Flushing d0d86132b03ea6f9edcee6d7811b1f14/a: closing flushed file at 1733732354745 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46c4d875: reopening flushed file at 1733732354797 (+52 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for d0d86132b03ea6f9edcee6d7811b1f14 in 103ms, sequenceid=2, compaction requested=false; wal=null at 1733732354815 (+18 ms)Cleaning up temporary data from old regions at 1733732354818 (+3 ms)Region opened successfully at 1733732354828 (+10 ms) 2024-12-09T08:19:14,860 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=372 (was 361) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:60250 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:38622 [Receiving block BP-1712704723-172.17.0.2-1733732346335:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:43926 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:45647/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1712704723-172.17.0.2-1733732346335:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:60390 [Receiving block BP-1712704723-172.17.0.2-1733732346335:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1712704723-172.17.0.2-1733732346335:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:38444 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1712704723-172.17.0.2-1733732346335:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:44054 [Receiving block BP-1712704723-172.17.0.2-1733732346335:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=271 (was 271), ProcessCount=11 (was 11), AvailableMemoryMB=4768 (was 4790) 2024-12-09T08:19:14,871 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=372, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=271, ProcessCount=11, AvailableMemoryMB=4766 2024-12-09T08:19:14,885 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:14,887 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:14,888 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:19:14,891 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-27438115, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/hregion-27438115, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:14,903 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-27438115/hregion-27438115.1733732354892, exclude list is [], retry=0 2024-12-09T08:19:14,907 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:14,907 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:14,907 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:14,934 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-27438115/hregion-27438115.1733732354892 2024-12-09T08:19:14,935 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41283:41283),(127.0.0.1/127.0.0.1:42925:42925),(127.0.0.1/127.0.0.1:42683:42683)] 2024-12-09T08:19:14,935 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => db9962709f4123f70aa7705f73e56f7a, NAME => 'testReplayEditsWrittenIntoWAL,,1733732354885.db9962709f4123f70aa7705f73e56f7a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45647/hbase 2024-12-09T08:19:14,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741846_1022 (size=64) 2024-12-09T08:19:14,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741846_1022 (size=64) 2024-12-09T08:19:14,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741846_1022 (size=64) 2024-12-09T08:19:14,949 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733732354885.db9962709f4123f70aa7705f73e56f7a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:14,951 INFO [StoreOpener-db9962709f4123f70aa7705f73e56f7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region db9962709f4123f70aa7705f73e56f7a 2024-12-09T08:19:14,954 INFO [StoreOpener-db9962709f4123f70aa7705f73e56f7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db9962709f4123f70aa7705f73e56f7a columnFamilyName a 2024-12-09T08:19:14,954 DEBUG [StoreOpener-db9962709f4123f70aa7705f73e56f7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:14,955 INFO [StoreOpener-db9962709f4123f70aa7705f73e56f7a-1 {}] regionserver.HStore(327): Store=db9962709f4123f70aa7705f73e56f7a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:14,955 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for db9962709f4123f70aa7705f73e56f7a 2024-12-09T08:19:14,956 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a 2024-12-09T08:19:14,957 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a 2024-12-09T08:19:14,958 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for db9962709f4123f70aa7705f73e56f7a 2024-12-09T08:19:14,958 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for db9962709f4123f70aa7705f73e56f7a 2024-12-09T08:19:14,961 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for db9962709f4123f70aa7705f73e56f7a 2024-12-09T08:19:14,964 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:19:14,965 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened db9962709f4123f70aa7705f73e56f7a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67160370, jitterRate=7.674992084503174E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T08:19:14,965 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for db9962709f4123f70aa7705f73e56f7a: Writing region info on filesystem at 1733732354950Initializing all the Stores at 1733732354951 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732354951Cleaning up temporary data from old regions at 1733732354958 (+7 ms)Region opened successfully at 1733732354965 (+7 ms) 2024-12-09T08:19:14,965 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing db9962709f4123f70aa7705f73e56f7a, disabling compactions & flushes 2024-12-09T08:19:14,966 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733732354885.db9962709f4123f70aa7705f73e56f7a. 2024-12-09T08:19:14,966 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733732354885.db9962709f4123f70aa7705f73e56f7a. 2024-12-09T08:19:14,966 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733732354885.db9962709f4123f70aa7705f73e56f7a. after waiting 0 ms 2024-12-09T08:19:14,966 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733732354885.db9962709f4123f70aa7705f73e56f7a. 2024-12-09T08:19:14,966 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733732354885.db9962709f4123f70aa7705f73e56f7a. 2024-12-09T08:19:14,966 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for db9962709f4123f70aa7705f73e56f7a: Waiting for close lock at 1733732354965Disabling compacts and flushes for region at 1733732354965Disabling writes for close at 1733732354966 (+1 ms)Writing region close event to WAL at 1733732354966Closed at 1733732354966 2024-12-09T08:19:14,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741845_1021 (size=175) 2024-12-09T08:19:14,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741845_1021 (size=175) 2024-12-09T08:19:14,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741845_1021 (size=175) 2024-12-09T08:19:14,973 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:19:14,973 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-27438115:(num 1733732354892) 2024-12-09T08:19:14,974 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T08:19:15,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741847_1023 (size=469) 2024-12-09T08:19:15,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741847_1023 (size=469) 2024-12-09T08:19:15,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741847_1023 (size=469) 2024-12-09T08:19:15,010 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T08:19:15,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741848_1024 (size=322) 2024-12-09T08:19:15,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741848_1024 (size=322) 2024-12-09T08:19:15,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741848_1024 (size=322) 2024-12-09T08:19:15,064 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884/wal-2, size=322 (322bytes) 2024-12-09T08:19:15,064 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884/wal-2 2024-12-09T08:19:15,065 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884/wal-2 after 1ms 2024-12-09T08:19:15,089 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:15,089 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884/wal-2 took 25ms 2024-12-09T08:19:15,092 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884/wal-2 so closing down 2024-12-09T08:19:15,092 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:19:15,094 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-09T08:19:15,116 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/recovered.edits/0000000000000000002-wal-2.temp 2024-12-09T08:19:15,117 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:19:15,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741849_1025 (size=322) 2024-12-09T08:19:15,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741849_1025 (size=322) 2024-12-09T08:19:15,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741849_1025 (size=322) 2024-12-09T08:19:15,128 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-09T08:19:15,131 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/recovered.edits/0000000000000000002 2024-12-09T08:19:15,131 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-09T08:19:15,131 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884/wal-2, journal: Splitting hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884/wal-2, size=322 (322bytes) at 1733732355064Finishing writing output for hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884/wal-2 so closing down at 1733732355092 (+28 ms)Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/recovered.edits/0000000000000000002-wal-2.temp at 1733732355116 (+24 ms)3 split writer threads finished at 1733732355117 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733732355129 (+12 ms)Rename recovered edits hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/recovered.edits/0000000000000000002 at 1733732355131 (+2 ms)Processed 1 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733732355131 2024-12-09T08:19:15,144 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884/wal-1, size=469 (469bytes) 2024-12-09T08:19:15,144 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884/wal-1 2024-12-09T08:19:15,145 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884/wal-1 after 1ms 2024-12-09T08:19:15,169 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:15,169 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884/wal-1 took 25ms 2024-12-09T08:19:15,171 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884/wal-1 so closing down 2024-12-09T08:19:15,171 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:19:15,173 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-09T08:19:15,197 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/recovered.edits/0000000000000000001-wal-1.temp 2024-12-09T08:19:15,198 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:19:15,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741850_1026 (size=469) 2024-12-09T08:19:15,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741850_1026 (size=469) 2024-12-09T08:19:15,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741850_1026 (size=469) 2024-12-09T08:19:15,206 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-09T08:19:15,236 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:15,239 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/recovered.edits/0000000000000000002, length=322 2024-12-09T08:19:15,241 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/recovered.edits/0000000000000000002 2024-12-09T08:19:15,242 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 72 ms; skipped=0; WAL=hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-09T08:19:15,242 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884/wal-1, journal: Splitting hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884/wal-1, size=469 (469bytes) at 1733732355144Finishing writing output for hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884/wal-1 so closing down at 1733732355171 (+27 ms)Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/recovered.edits/0000000000000000001-wal-1.temp at 1733732355197 (+26 ms)3 split writer threads finished at 1733732355198 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733732355206 (+8 ms)Rename recovered edits hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/recovered.edits/0000000000000000002 at 1733732355241 (+35 ms)Processed 2 edits across 1 Regions in 72 ms; skipped=0; WAL=hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733732355242 (+1 ms) 2024-12-09T08:19:15,242 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:19:15,244 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:15,258 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884/wal.1733732355245, exclude list is [], retry=0 2024-12-09T08:19:15,261 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:15,261 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:15,261 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:15,295 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884/wal.1733732355245 2024-12-09T08:19:15,296 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42683:42683),(127.0.0.1/127.0.0.1:41283:41283),(127.0.0.1/127.0.0.1:42925:42925)] 2024-12-09T08:19:15,296 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => db9962709f4123f70aa7705f73e56f7a, NAME => 'testReplayEditsWrittenIntoWAL,,1733732354885.db9962709f4123f70aa7705f73e56f7a.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:19:15,296 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733732354885.db9962709f4123f70aa7705f73e56f7a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:15,296 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for db9962709f4123f70aa7705f73e56f7a 2024-12-09T08:19:15,296 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for db9962709f4123f70aa7705f73e56f7a 2024-12-09T08:19:15,298 INFO [StoreOpener-db9962709f4123f70aa7705f73e56f7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region db9962709f4123f70aa7705f73e56f7a 2024-12-09T08:19:15,300 INFO [StoreOpener-db9962709f4123f70aa7705f73e56f7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db9962709f4123f70aa7705f73e56f7a columnFamilyName a 2024-12-09T08:19:15,300 DEBUG [StoreOpener-db9962709f4123f70aa7705f73e56f7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:15,301 INFO [StoreOpener-db9962709f4123f70aa7705f73e56f7a-1 {}] regionserver.HStore(327): Store=db9962709f4123f70aa7705f73e56f7a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:15,301 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for db9962709f4123f70aa7705f73e56f7a 2024-12-09T08:19:15,302 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a 2024-12-09T08:19:15,304 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a 2024-12-09T08:19:15,305 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/recovered.edits/0000000000000000002 2024-12-09T08:19:15,332 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:15,334 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/recovered.edits/0000000000000000002 2024-12-09T08:19:15,335 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing db9962709f4123f70aa7705f73e56f7a 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-09T08:19:15,355 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/.tmp/a/0642731f051048d9b1efb980513449a4 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733732354973/Put/seqid=0 2024-12-09T08:19:15,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741852_1028 (size=5170) 2024-12-09T08:19:15,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741852_1028 (size=5170) 2024-12-09T08:19:15,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741852_1028 (size=5170) 2024-12-09T08:19:15,364 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/.tmp/a/0642731f051048d9b1efb980513449a4 2024-12-09T08:19:15,372 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/.tmp/a/0642731f051048d9b1efb980513449a4 as hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/a/0642731f051048d9b1efb980513449a4 2024-12-09T08:19:15,381 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/a/0642731f051048d9b1efb980513449a4, entries=2, sequenceid=2, filesize=5.0 K 2024-12-09T08:19:15,381 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for db9962709f4123f70aa7705f73e56f7a in 46ms, sequenceid=2, compaction requested=false; wal=null 2024-12-09T08:19:15,382 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/recovered.edits/0000000000000000002 2024-12-09T08:19:15,383 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for db9962709f4123f70aa7705f73e56f7a 2024-12-09T08:19:15,383 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for db9962709f4123f70aa7705f73e56f7a 2024-12-09T08:19:15,386 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for db9962709f4123f70aa7705f73e56f7a 2024-12-09T08:19:15,389 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/db9962709f4123f70aa7705f73e56f7a/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-09T08:19:15,390 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened db9962709f4123f70aa7705f73e56f7a; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74128841, jitterRate=0.10460580885410309}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T08:19:15,390 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for db9962709f4123f70aa7705f73e56f7a: Writing region info on filesystem at 1733732355296Initializing all the Stores at 1733732355298 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732355298Obtaining lock to block concurrent updates at 1733732355335 (+37 ms)Preparing flush snapshotting stores in db9962709f4123f70aa7705f73e56f7a at 1733732355335Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733732354885.db9962709f4123f70aa7705f73e56f7a., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733732355335Flushing stores of testReplayEditsWrittenIntoWAL,,1733732354885.db9962709f4123f70aa7705f73e56f7a. at 1733732355335Flushing db9962709f4123f70aa7705f73e56f7a/a: creating writer at 1733732355335Flushing db9962709f4123f70aa7705f73e56f7a/a: appending metadata at 1733732355353 (+18 ms)Flushing db9962709f4123f70aa7705f73e56f7a/a: closing flushed file at 1733732355354 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@249a9509: reopening flushed file at 1733732355371 (+17 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for db9962709f4123f70aa7705f73e56f7a in 46ms, sequenceid=2, compaction requested=false; wal=null at 1733732355381 (+10 ms)Cleaning up temporary data from old regions at 1733732355383 (+2 ms)Region opened successfully at 1733732355390 (+7 ms) 2024-12-09T08:19:15,404 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=382 (was 372) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:44118 [Receiving block BP-1712704723-172.17.0.2-1733732346335:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:43926 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1712704723-172.17.0.2-1733732346335:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:60464 [Receiving block BP-1712704723-172.17.0.2-1733732346335:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1712704723-172.17.0.2-1733732346335:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1712704723-172.17.0.2-1733732346335:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:38702 [Receiving block BP-1712704723-172.17.0.2-1733732346335:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:38444 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=271 (was 271), ProcessCount=11 (was 11), AvailableMemoryMB=4752 (was 4766) 2024-12-09T08:19:15,414 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=382, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=271, ProcessCount=11, AvailableMemoryMB=4751 2024-12-09T08:19:15,428 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:15,430 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:15,431 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:19:15,435 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-50436494, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/hregion-50436494, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:15,445 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-50436494/hregion-50436494.1733732355435, exclude list is [], retry=0 2024-12-09T08:19:15,449 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:15,449 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:15,449 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:15,471 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-50436494/hregion-50436494.1733732355435 2024-12-09T08:19:15,471 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42925:42925),(127.0.0.1/127.0.0.1:42683:42683),(127.0.0.1/127.0.0.1:41283:41283)] 2024-12-09T08:19:15,471 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2b6ab886b96188eb28b4a7480849705d, NAME => 'testReplayEditsWrittenIntoWAL,,1733732355429.2b6ab886b96188eb28b4a7480849705d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45647/hbase 2024-12-09T08:19:15,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741854_1030 (size=64) 2024-12-09T08:19:15,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741854_1030 (size=64) 2024-12-09T08:19:15,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741854_1030 (size=64) 2024-12-09T08:19:15,485 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733732355429.2b6ab886b96188eb28b4a7480849705d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:15,487 INFO [StoreOpener-2b6ab886b96188eb28b4a7480849705d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2b6ab886b96188eb28b4a7480849705d 2024-12-09T08:19:15,489 INFO [StoreOpener-2b6ab886b96188eb28b4a7480849705d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b6ab886b96188eb28b4a7480849705d columnFamilyName a 2024-12-09T08:19:15,489 DEBUG [StoreOpener-2b6ab886b96188eb28b4a7480849705d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:15,489 INFO [StoreOpener-2b6ab886b96188eb28b4a7480849705d-1 {}] regionserver.HStore(327): Store=2b6ab886b96188eb28b4a7480849705d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:15,489 INFO [StoreOpener-2b6ab886b96188eb28b4a7480849705d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2b6ab886b96188eb28b4a7480849705d 2024-12-09T08:19:15,491 INFO [StoreOpener-2b6ab886b96188eb28b4a7480849705d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b6ab886b96188eb28b4a7480849705d columnFamilyName b 2024-12-09T08:19:15,491 DEBUG [StoreOpener-2b6ab886b96188eb28b4a7480849705d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:15,492 INFO [StoreOpener-2b6ab886b96188eb28b4a7480849705d-1 {}] regionserver.HStore(327): Store=2b6ab886b96188eb28b4a7480849705d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:15,492 INFO [StoreOpener-2b6ab886b96188eb28b4a7480849705d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2b6ab886b96188eb28b4a7480849705d 2024-12-09T08:19:15,493 INFO [StoreOpener-2b6ab886b96188eb28b4a7480849705d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b6ab886b96188eb28b4a7480849705d columnFamilyName c 2024-12-09T08:19:15,493 DEBUG [StoreOpener-2b6ab886b96188eb28b4a7480849705d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:15,494 INFO [StoreOpener-2b6ab886b96188eb28b4a7480849705d-1 {}] regionserver.HStore(327): Store=2b6ab886b96188eb28b4a7480849705d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:15,494 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2b6ab886b96188eb28b4a7480849705d 2024-12-09T08:19:15,495 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d 2024-12-09T08:19:15,495 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d 2024-12-09T08:19:15,497 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2b6ab886b96188eb28b4a7480849705d 2024-12-09T08:19:15,497 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2b6ab886b96188eb28b4a7480849705d 2024-12-09T08:19:15,497 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:19:15,498 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2b6ab886b96188eb28b4a7480849705d 2024-12-09T08:19:15,501 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:19:15,501 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2b6ab886b96188eb28b4a7480849705d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74443877, jitterRate=0.10930021107196808}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:19:15,502 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2b6ab886b96188eb28b4a7480849705d: Writing region info on filesystem at 1733732355485Initializing all the Stores at 1733732355486 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732355486Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732355487 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732355487Cleaning up temporary data from old regions at 1733732355497 (+10 ms)Region opened successfully at 1733732355502 (+5 ms) 2024-12-09T08:19:15,502 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2b6ab886b96188eb28b4a7480849705d, disabling compactions & flushes 2024-12-09T08:19:15,502 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733732355429.2b6ab886b96188eb28b4a7480849705d. 2024-12-09T08:19:15,502 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733732355429.2b6ab886b96188eb28b4a7480849705d. 2024-12-09T08:19:15,502 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733732355429.2b6ab886b96188eb28b4a7480849705d. after waiting 0 ms 2024-12-09T08:19:15,502 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733732355429.2b6ab886b96188eb28b4a7480849705d. 2024-12-09T08:19:15,502 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733732355429.2b6ab886b96188eb28b4a7480849705d. 2024-12-09T08:19:15,503 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2b6ab886b96188eb28b4a7480849705d: Waiting for close lock at 1733732355502Disabling compacts and flushes for region at 1733732355502Disabling writes for close at 1733732355502Writing region close event to WAL at 1733732355502Closed at 1733732355502 2024-12-09T08:19:15,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741853_1029 (size=175) 2024-12-09T08:19:15,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741853_1029 (size=175) 2024-12-09T08:19:15,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741853_1029 (size=175) 2024-12-09T08:19:15,509 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:19:15,509 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-50436494:(num 1733732355435) 2024-12-09T08:19:15,509 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:19:15,512 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732355427, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:15,525 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732355427/wal.1733732355513, exclude list is [], retry=0 2024-12-09T08:19:15,528 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:15,529 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:15,529 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:15,556 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732355427/wal.1733732355513 2024-12-09T08:19:15,556 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42683:42683),(127.0.0.1/127.0.0.1:42925:42925),(127.0.0.1/127.0.0.1:41283:41283)] 2024-12-09T08:19:15,727 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732355427/wal.1733732355513, size=0 (0bytes) 2024-12-09T08:19:15,727 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732355427/wal.1733732355513 might be still open, length is 0 2024-12-09T08:19:15,727 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732355427/wal.1733732355513 2024-12-09T08:19:15,728 WARN [IPC Server handler 2 on default port 45647 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732355427/wal.1733732355513 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-09T08:19:15,729 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732355427/wal.1733732355513 after 2ms 2024-12-09T08:19:15,941 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:60502 [Receiving block BP-1712704723-172.17.0.2-1733732346335:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:38689:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60502 dst: /127.0.0.1:38689 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:38689 remote=/127.0.0.1:60502]. Total timeout mills is 60000, 59759 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:15,942 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:44154 [Receiving block BP-1712704723-172.17.0.2-1733732346335:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:45463:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44154 dst: /127.0.0.1:45463 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:15,942 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:38726 [Receiving block BP-1712704723-172.17.0.2-1733732346335:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41399:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38726 dst: /127.0.0.1:41399 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:15,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741855_1032 (size=497775) 2024-12-09T08:19:15,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741855_1032 (size=497775) 2024-12-09T08:19:15,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741855_1032 (size=497775) 2024-12-09T08:19:19,113 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T08:19:19,166 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-09T08:19:19,731 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732355427/wal.1733732355513 after 4004ms 2024-12-09T08:19:19,767 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:19,769 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732355427/wal.1733732355513 took 4042ms 2024-12-09T08:19:19,773 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733732355513.temp 2024-12-09T08:19:19,814 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/recovered.edits/0000000000000000001-wal.1733732355513.temp 2024-12-09T08:19:19,866 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732355427/wal.1733732355513; continuing. 2024-12-09T08:19:19,866 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732355427/wal.1733732355513 so closing down 2024-12-09T08:19:19,866 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:19:19,867 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:19:19,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741856_1033 (size=497783) 2024-12-09T08:19:19,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741856_1033 (size=497783) 2024-12-09T08:19:19,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741856_1033 (size=497783) 2024-12-09T08:19:19,871 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/recovered.edits/0000000000000000001-wal.1733732355513.temp (wrote 3002 edits, skipped 0 edits in 42 ms) 2024-12-09T08:19:19,872 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/recovered.edits/0000000000000000001-wal.1733732355513.temp to hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/recovered.edits/0000000000000003002 2024-12-09T08:19:19,872 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 103 ms; skipped=0; WAL=hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732355427/wal.1733732355513, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T08:19:19,872 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732355427/wal.1733732355513, journal: Splitting hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732355427/wal.1733732355513, size=0 (0bytes) at 1733732355727Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/recovered.edits/0000000000000000001-wal.1733732355513.temp at 1733732359814 (+4087 ms)Split 1024 edits, skipped 0 edits. at 1733732359823 (+9 ms)Split 2048 edits, skipped 0 edits. at 1733732359849 (+26 ms)Finishing writing output for hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732355427/wal.1733732355513 so closing down at 1733732359866 (+17 ms)3 split writer threads finished at 1733732359867 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/recovered.edits/0000000000000000001-wal.1733732355513.temp (wrote 3002 edits, skipped 0 edits in 42 ms) at 1733732359871 (+4 ms)Rename recovered edits hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/recovered.edits/0000000000000000001-wal.1733732355513.temp to hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/recovered.edits/0000000000000003002 at 1733732359872 (+1 ms)Processed 3002 edits across 1 Regions in 103 ms; skipped=0; WAL=hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732355427/wal.1733732355513, size=0, length=0, corrupted=false, cancelled=false at 1733732359872 2024-12-09T08:19:19,874 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732355427/wal.1733732355513 to hdfs://localhost:45647/hbase/oldWALs/wal.1733732355513 2024-12-09T08:19:19,876 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/recovered.edits/0000000000000003002 2024-12-09T08:19:19,876 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:19:19,879 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732355427, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:19,892 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732355427/wal.1733732359880, exclude list is [], retry=0 2024-12-09T08:19:19,895 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:19,895 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:19,896 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:19,920 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732355427/wal.1733732359880 2024-12-09T08:19:19,921 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42683:42683),(127.0.0.1/127.0.0.1:42925:42925),(127.0.0.1/127.0.0.1:41283:41283)] 2024-12-09T08:19:19,921 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733732355429.2b6ab886b96188eb28b4a7480849705d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:19,923 INFO [StoreOpener-2b6ab886b96188eb28b4a7480849705d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2b6ab886b96188eb28b4a7480849705d 2024-12-09T08:19:19,924 INFO [StoreOpener-2b6ab886b96188eb28b4a7480849705d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b6ab886b96188eb28b4a7480849705d columnFamilyName a 2024-12-09T08:19:19,924 DEBUG [StoreOpener-2b6ab886b96188eb28b4a7480849705d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:19,925 INFO [StoreOpener-2b6ab886b96188eb28b4a7480849705d-1 {}] regionserver.HStore(327): Store=2b6ab886b96188eb28b4a7480849705d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:19,925 INFO [StoreOpener-2b6ab886b96188eb28b4a7480849705d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2b6ab886b96188eb28b4a7480849705d 2024-12-09T08:19:19,926 INFO [StoreOpener-2b6ab886b96188eb28b4a7480849705d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b6ab886b96188eb28b4a7480849705d columnFamilyName b 2024-12-09T08:19:19,926 DEBUG [StoreOpener-2b6ab886b96188eb28b4a7480849705d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:19,927 INFO [StoreOpener-2b6ab886b96188eb28b4a7480849705d-1 {}] regionserver.HStore(327): Store=2b6ab886b96188eb28b4a7480849705d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:19,927 INFO [StoreOpener-2b6ab886b96188eb28b4a7480849705d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2b6ab886b96188eb28b4a7480849705d 2024-12-09T08:19:19,928 INFO [StoreOpener-2b6ab886b96188eb28b4a7480849705d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b6ab886b96188eb28b4a7480849705d columnFamilyName c 2024-12-09T08:19:19,928 DEBUG [StoreOpener-2b6ab886b96188eb28b4a7480849705d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:19,929 INFO [StoreOpener-2b6ab886b96188eb28b4a7480849705d-1 {}] regionserver.HStore(327): Store=2b6ab886b96188eb28b4a7480849705d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:19,929 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2b6ab886b96188eb28b4a7480849705d 2024-12-09T08:19:19,930 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d 2024-12-09T08:19:19,932 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d 2024-12-09T08:19:19,933 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/recovered.edits/0000000000000003002 2024-12-09T08:19:19,958 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:19,991 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T08:19:20,296 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2b6ab886b96188eb28b4a7480849705d 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-09T08:19:20,332 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/.tmp/a/64b863bc8a7141e88765a2de872c5894 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733732355563/Put/seqid=0 2024-12-09T08:19:20,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741858_1035 (size=50463) 2024-12-09T08:19:20,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741858_1035 (size=50463) 2024-12-09T08:19:20,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741858_1035 (size=50463) 2024-12-09T08:19:20,349 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/.tmp/a/64b863bc8a7141e88765a2de872c5894 2024-12-09T08:19:20,358 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/.tmp/a/64b863bc8a7141e88765a2de872c5894 as hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/a/64b863bc8a7141e88765a2de872c5894 2024-12-09T08:19:20,365 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/a/64b863bc8a7141e88765a2de872c5894, entries=754, sequenceid=754, filesize=49.3 K 2024-12-09T08:19:20,366 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 2b6ab886b96188eb28b4a7480849705d in 70ms, sequenceid=754, compaction requested=false; wal=null 2024-12-09T08:19:20,389 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T08:19:20,389 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2b6ab886b96188eb28b4a7480849705d 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-09T08:19:20,399 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/.tmp/a/a597eeb9b7cd402687f77b3023c5b55c is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733732355598/Put/seqid=0 2024-12-09T08:19:20,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741859_1036 (size=20072) 2024-12-09T08:19:20,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741859_1036 (size=20072) 2024-12-09T08:19:20,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741859_1036 (size=20072) 2024-12-09T08:19:20,408 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/.tmp/a/a597eeb9b7cd402687f77b3023c5b55c 2024-12-09T08:19:20,434 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/.tmp/b/31f8e43999c04d4ba9bc77f4b43ef9d5 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733732355619/Put/seqid=0 2024-12-09T08:19:20,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741860_1037 (size=35835) 2024-12-09T08:19:20,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741860_1037 (size=35835) 2024-12-09T08:19:20,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741860_1037 (size=35835) 2024-12-09T08:19:20,442 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/.tmp/b/31f8e43999c04d4ba9bc77f4b43ef9d5 2024-12-09T08:19:20,450 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/.tmp/a/a597eeb9b7cd402687f77b3023c5b55c as hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/a/a597eeb9b7cd402687f77b3023c5b55c 2024-12-09T08:19:20,458 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/a/a597eeb9b7cd402687f77b3023c5b55c, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-09T08:19:20,460 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/.tmp/b/31f8e43999c04d4ba9bc77f4b43ef9d5 as hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/b/31f8e43999c04d4ba9bc77f4b43ef9d5 2024-12-09T08:19:20,467 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/b/31f8e43999c04d4ba9bc77f4b43ef9d5, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-09T08:19:20,467 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 2b6ab886b96188eb28b4a7480849705d in 78ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-09T08:19:20,477 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T08:19:20,477 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2b6ab886b96188eb28b4a7480849705d 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-09T08:19:20,483 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/.tmp/b/2799894ea2af4cf88f8fc9d0e9fea5ab is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733732355634/Put/seqid=0 2024-12-09T08:19:20,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741861_1038 (size=35082) 2024-12-09T08:19:20,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741861_1038 (size=35082) 2024-12-09T08:19:20,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741861_1038 (size=35082) 2024-12-09T08:19:20,496 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/.tmp/b/2799894ea2af4cf88f8fc9d0e9fea5ab 2024-12-09T08:19:20,518 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/.tmp/c/a4c15e22adce44b29bf2f60ae06a7c42 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733732355664/Put/seqid=0 2024-12-09T08:19:20,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741862_1039 (size=20825) 2024-12-09T08:19:20,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741862_1039 (size=20825) 2024-12-09T08:19:20,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741862_1039 (size=20825) 2024-12-09T08:19:20,526 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/.tmp/c/a4c15e22adce44b29bf2f60ae06a7c42 2024-12-09T08:19:20,534 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/.tmp/b/2799894ea2af4cf88f8fc9d0e9fea5ab as hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/b/2799894ea2af4cf88f8fc9d0e9fea5ab 2024-12-09T08:19:20,541 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/b/2799894ea2af4cf88f8fc9d0e9fea5ab, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-09T08:19:20,542 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/.tmp/c/a4c15e22adce44b29bf2f60ae06a7c42 as hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/c/a4c15e22adce44b29bf2f60ae06a7c42 2024-12-09T08:19:20,550 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/c/a4c15e22adce44b29bf2f60ae06a7c42, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-09T08:19:20,550 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 2b6ab886b96188eb28b4a7480849705d in 73ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-09T08:19:20,565 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733732355698/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733732355429.2b6ab886b96188eb28b4a7480849705d. 2024-12-09T08:19:20,569 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/recovered.edits/0000000000000003002 2024-12-09T08:19:20,569 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T08:19:20,570 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2b6ab886b96188eb28b4a7480849705d 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-09T08:19:20,578 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/.tmp/c/1577ef30200746c0a53fc02e5645f0bd is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733732355669/Put/seqid=0 2024-12-09T08:19:20,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741863_1040 (size=50301) 2024-12-09T08:19:20,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741863_1040 (size=50301) 2024-12-09T08:19:20,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741863_1040 (size=50301) 2024-12-09T08:19:20,587 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/.tmp/c/1577ef30200746c0a53fc02e5645f0bd 2024-12-09T08:19:20,594 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1577ef30200746c0a53fc02e5645f0bd 2024-12-09T08:19:20,596 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/.tmp/c/1577ef30200746c0a53fc02e5645f0bd as hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/c/1577ef30200746c0a53fc02e5645f0bd 2024-12-09T08:19:20,603 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1577ef30200746c0a53fc02e5645f0bd 2024-12-09T08:19:20,603 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/c/1577ef30200746c0a53fc02e5645f0bd, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-09T08:19:20,604 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 2b6ab886b96188eb28b4a7480849705d in 35ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-09T08:19:20,605 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/recovered.edits/0000000000000003002 2024-12-09T08:19:20,607 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2b6ab886b96188eb28b4a7480849705d 2024-12-09T08:19:20,607 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2b6ab886b96188eb28b4a7480849705d 2024-12-09T08:19:20,608 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T08:19:20,610 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2b6ab886b96188eb28b4a7480849705d 2024-12-09T08:19:20,613 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenIntoWAL/2b6ab886b96188eb28b4a7480849705d/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-09T08:19:20,614 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2b6ab886b96188eb28b4a7480849705d; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64409868, jitterRate=-0.04021817445755005}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T08:19:20,614 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2b6ab886b96188eb28b4a7480849705d: Writing region info on filesystem at 1733732359921Initializing all the Stores at 1733732359923 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732359923Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732359923Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732359923Cleaning up temporary data from old regions at 1733732360607 (+684 ms)Region opened successfully at 1733732360614 (+7 ms) 2024-12-09T08:19:20,668 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2b6ab886b96188eb28b4a7480849705d, disabling compactions & flushes 2024-12-09T08:19:20,668 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733732355429.2b6ab886b96188eb28b4a7480849705d. 2024-12-09T08:19:20,668 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733732355429.2b6ab886b96188eb28b4a7480849705d. 2024-12-09T08:19:20,668 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733732355429.2b6ab886b96188eb28b4a7480849705d. after waiting 0 ms 2024-12-09T08:19:20,668 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733732355429.2b6ab886b96188eb28b4a7480849705d. 2024-12-09T08:19:20,670 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733732355429.2b6ab886b96188eb28b4a7480849705d. 2024-12-09T08:19:20,670 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2b6ab886b96188eb28b4a7480849705d: Waiting for close lock at 1733732360668Disabling compacts and flushes for region at 1733732360668Disabling writes for close at 1733732360668Writing region close event to WAL at 1733732360670 (+2 ms)Closed at 1733732360670 2024-12-09T08:19:20,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741857_1034 (size=175) 2024-12-09T08:19:20,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741857_1034 (size=175) 2024-12-09T08:19:20,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741857_1034 (size=175) 2024-12-09T08:19:20,676 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:19:20,677 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733732359880) 2024-12-09T08:19:20,690 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=396 (was 382) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/cluster_95cc11f3-affe-6e07-bfcd-38f5721bf4f5/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-614128667_22 at /127.0.0.1:57152 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34795 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-614128667_22 at /127.0.0.1:55178 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:34795 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42153 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:42153 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-614128667_22 at /127.0.0.1:40312 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/cluster_95cc11f3-affe-6e07-bfcd-38f5721bf4f5/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-614128667_22 at /127.0.0.1:57140 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:45647 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:45647 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=250 (was 271), ProcessCount=11 (was 11), AvailableMemoryMB=4650 (was 4751) 2024-12-09T08:19:20,700 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=396, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=250, ProcessCount=11, AvailableMemoryMB=4649 2024-12-09T08:19:20,714 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:20,716 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:20,717 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:19:20,720 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-67630778, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/hregion-67630778, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:20,731 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-67630778/hregion-67630778.1733732360721, exclude list is [], retry=0 2024-12-09T08:19:20,734 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:20,734 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:20,734 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:20,759 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-67630778/hregion-67630778.1733732360721 2024-12-09T08:19:20,759 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42925:42925),(127.0.0.1/127.0.0.1:42683:42683),(127.0.0.1/127.0.0.1:41283:41283)] 2024-12-09T08:19:20,759 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ee76d2275a7974070cadc12d861d255e, NAME => 'test2727,,1733732360715.ee76d2275a7974070cadc12d861d255e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45647/hbase 2024-12-09T08:19:20,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741865_1042 (size=43) 2024-12-09T08:19:20,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741865_1042 (size=43) 2024-12-09T08:19:20,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741865_1042 (size=43) 2024-12-09T08:19:20,770 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733732360715.ee76d2275a7974070cadc12d861d255e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:20,772 INFO [StoreOpener-ee76d2275a7974070cadc12d861d255e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ee76d2275a7974070cadc12d861d255e 2024-12-09T08:19:20,773 INFO [StoreOpener-ee76d2275a7974070cadc12d861d255e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee76d2275a7974070cadc12d861d255e columnFamilyName a 2024-12-09T08:19:20,773 DEBUG [StoreOpener-ee76d2275a7974070cadc12d861d255e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:20,774 INFO [StoreOpener-ee76d2275a7974070cadc12d861d255e-1 {}] regionserver.HStore(327): Store=ee76d2275a7974070cadc12d861d255e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:20,774 INFO [StoreOpener-ee76d2275a7974070cadc12d861d255e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ee76d2275a7974070cadc12d861d255e 2024-12-09T08:19:20,775 INFO [StoreOpener-ee76d2275a7974070cadc12d861d255e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee76d2275a7974070cadc12d861d255e columnFamilyName b 2024-12-09T08:19:20,775 DEBUG [StoreOpener-ee76d2275a7974070cadc12d861d255e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:20,775 INFO [StoreOpener-ee76d2275a7974070cadc12d861d255e-1 {}] regionserver.HStore(327): Store=ee76d2275a7974070cadc12d861d255e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:20,776 INFO [StoreOpener-ee76d2275a7974070cadc12d861d255e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ee76d2275a7974070cadc12d861d255e 2024-12-09T08:19:20,777 INFO [StoreOpener-ee76d2275a7974070cadc12d861d255e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee76d2275a7974070cadc12d861d255e columnFamilyName c 2024-12-09T08:19:20,777 DEBUG [StoreOpener-ee76d2275a7974070cadc12d861d255e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:20,778 INFO [StoreOpener-ee76d2275a7974070cadc12d861d255e-1 {}] regionserver.HStore(327): Store=ee76d2275a7974070cadc12d861d255e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:20,778 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ee76d2275a7974070cadc12d861d255e 2024-12-09T08:19:20,779 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e 2024-12-09T08:19:20,779 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e 2024-12-09T08:19:20,780 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ee76d2275a7974070cadc12d861d255e 2024-12-09T08:19:20,780 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ee76d2275a7974070cadc12d861d255e 2024-12-09T08:19:20,781 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:19:20,782 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ee76d2275a7974070cadc12d861d255e 2024-12-09T08:19:20,787 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:19:20,788 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ee76d2275a7974070cadc12d861d255e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65772695, jitterRate=-0.01991046965122223}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:19:20,789 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ee76d2275a7974070cadc12d861d255e: Writing region info on filesystem at 1733732360770Initializing all the Stores at 1733732360771 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732360771Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732360771Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732360771Cleaning up temporary data from old regions at 1733732360780 (+9 ms)Region opened successfully at 1733732360789 (+9 ms) 2024-12-09T08:19:20,789 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ee76d2275a7974070cadc12d861d255e, disabling compactions & flushes 2024-12-09T08:19:20,790 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733732360715.ee76d2275a7974070cadc12d861d255e. 2024-12-09T08:19:20,790 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733732360715.ee76d2275a7974070cadc12d861d255e. 2024-12-09T08:19:20,790 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733732360715.ee76d2275a7974070cadc12d861d255e. after waiting 0 ms 2024-12-09T08:19:20,790 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733732360715.ee76d2275a7974070cadc12d861d255e. 2024-12-09T08:19:20,790 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733732360715.ee76d2275a7974070cadc12d861d255e. 2024-12-09T08:19:20,790 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ee76d2275a7974070cadc12d861d255e: Waiting for close lock at 1733732360789Disabling compacts and flushes for region at 1733732360789Disabling writes for close at 1733732360790 (+1 ms)Writing region close event to WAL at 1733732360790Closed at 1733732360790 2024-12-09T08:19:20,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741864_1041 (size=175) 2024-12-09T08:19:20,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741864_1041 (size=175) 2024-12-09T08:19:20,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741864_1041 (size=175) 2024-12-09T08:19:20,797 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:19:20,797 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-67630778:(num 1733732360721) 2024-12-09T08:19:20,797 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:19:20,799 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/test2727-manual,16010,1733732360713, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:20,812 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732360799, exclude list is [], retry=0 2024-12-09T08:19:20,815 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:20,816 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:20,816 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:20,840 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732360799 2024-12-09T08:19:20,840 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42925:42925),(127.0.0.1/127.0.0.1:42683:42683),(127.0.0.1/127.0.0.1:41283:41283)] 2024-12-09T08:19:20,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741866_1043 (size=371388) 2024-12-09T08:19:20,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741866_1043 (size=371388) 2024-12-09T08:19:20,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741866_1043 (size=371388) 2024-12-09T08:19:20,986 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45647/hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732360799, size=362.7 K (371388bytes) 2024-12-09T08:19:20,986 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45647/hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732360799 2024-12-09T08:19:20,987 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45647/hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732360799 after 0ms 2024-12-09T08:19:21,026 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:21,027 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45647/hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732360799 took 41ms 2024-12-09T08:19:21,032 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733732360799.temp 2024-12-09T08:19:21,067 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-09T08:19:21,068 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-09T08:19:21,070 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T08:19:21,070 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-09T08:19:21,070 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-09T08:19:21,071 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-09T08:19:21,072 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-09T08:19:21,072 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-09T08:19:21,073 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45647/hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732360799 so closing down 2024-12-09T08:19:21,073 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:19:21,073 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/0000000000000000001-wal.1733732360799.temp 2024-12-09T08:19:21,095 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:19:21,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741867_1044 (size=371388) 2024-12-09T08:19:21,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741867_1044 (size=371388) 2024-12-09T08:19:21,100 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/0000000000000000001-wal.1733732360799.temp (wrote 3000 edits, skipped 0 edits in 21 ms) 2024-12-09T08:19:21,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741867_1044 (size=371388) 2024-12-09T08:19:21,102 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/0000000000000000001-wal.1733732360799.temp to hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/0000000000000003000 2024-12-09T08:19:21,102 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 74 ms; skipped=0; WAL=hdfs://localhost:45647/hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732360799, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-12-09T08:19:21,102 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45647/hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732360799, journal: Splitting hdfs://localhost:45647/hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732360799, size=362.7 K (371388bytes) at 1733732360986Split 1024 edits, skipped 0 edits. at 1733732361045 (+59 ms)Split 2048 edits, skipped 0 edits. at 1733732361059 (+14 ms)Finishing writing output for hdfs://localhost:45647/hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732360799 so closing down at 1733732361073 (+14 ms)Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/0000000000000000001-wal.1733732360799.temp at 17337323610733 split writer threads finished at 1733732361095 (+22 ms)Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/0000000000000000001-wal.1733732360799.temp (wrote 3000 edits, skipped 0 edits in 21 ms) at 1733732361100 (+5 ms)Rename recovered edits hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/0000000000000000001-wal.1733732360799.temp to hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/0000000000000003000 at 1733732361102 (+2 ms)Processed 3000 edits across 1 Regions in 74 ms; skipped=0; WAL=hdfs://localhost:45647/hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732360799, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1733732361102 2024-12-09T08:19:21,104 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45647/hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732360799 to hdfs://localhost:45647/hbase/oldWALs/wal.1733732360799 2024-12-09T08:19:21,105 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/0000000000000003000 2024-12-09T08:19:21,105 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:19:21,107 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/test2727-manual,16010,1733732360713, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:21,121 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732361108, exclude list is [], retry=0 2024-12-09T08:19:21,124 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:21,125 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:21,126 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:21,156 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732361108 2024-12-09T08:19:21,156 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42925:42925),(127.0.0.1/127.0.0.1:41283:41283),(127.0.0.1/127.0.0.1:42683:42683)] 2024-12-09T08:19:21,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741868_1045 (size=371515) 2024-12-09T08:19:21,294 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732361108 not finished, retry = 0 2024-12-09T08:19:21,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741868_1045 (size=371515) 2024-12-09T08:19:21,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741868_1045 (size=371515) 2024-12-09T08:19:21,409 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45647/hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732361108, size=362.8 K (371515bytes) 2024-12-09T08:19:21,410 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45647/hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732361108 2024-12-09T08:19:21,410 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45647/hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732361108 after 0ms 2024-12-09T08:19:21,435 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:21,436 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45647/hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732361108 took 27ms 2024-12-09T08:19:21,440 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733732361108.temp 2024-12-09T08:19:21,485 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45647/hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732361108 so closing down 2024-12-09T08:19:21,485 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:19:21,509 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/0000000000000003001-wal.1733732361108.temp 2024-12-09T08:19:21,522 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:19:21,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741869_1046 (size=371515) 2024-12-09T08:19:21,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741869_1046 (size=371515) 2024-12-09T08:19:21,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741869_1046 (size=371515) 2024-12-09T08:19:21,531 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/0000000000000003001-wal.1733732361108.temp (wrote 3000 edits, skipped 0 edits in 12 ms) 2024-12-09T08:19:21,532 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/0000000000000003001-wal.1733732361108.temp to hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/0000000000000006000 2024-12-09T08:19:21,533 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 95 ms; skipped=0; WAL=hdfs://localhost:45647/hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732361108, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-12-09T08:19:21,533 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45647/hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732361108, journal: Splitting hdfs://localhost:45647/hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732361108, size=362.8 K (371515bytes) at 1733732361409Split 1024 edits, skipped 0 edits. at 1733732361470 (+61 ms)Split 2048 edits, skipped 0 edits. at 1733732361477 (+7 ms)Finishing writing output for hdfs://localhost:45647/hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732361108 so closing down at 1733732361485 (+8 ms)Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/0000000000000003001-wal.1733732361108.temp at 1733732361509 (+24 ms)3 split writer threads finished at 1733732361522 (+13 ms)Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/0000000000000003001-wal.1733732361108.temp (wrote 3000 edits, skipped 0 edits in 12 ms) at 1733732361531 (+9 ms)Rename recovered edits hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/0000000000000003001-wal.1733732361108.temp to hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/0000000000000006000 at 1733732361532 (+1 ms)Processed 3000 edits across 1 Regions in 95 ms; skipped=0; WAL=hdfs://localhost:45647/hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732361108, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1733732361533 (+1 ms) 2024-12-09T08:19:21,534 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45647/hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732361108 to hdfs://localhost:45647/hbase/oldWALs/wal.1733732361108 2024-12-09T08:19:21,535 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/0000000000000006000 2024-12-09T08:19:21,535 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:19:21,537 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/test2727-manual,16010,1733732360713, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:21,549 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732361538, exclude list is [], retry=0 2024-12-09T08:19:21,552 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:21,552 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:21,553 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:21,578 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733732360713/wal.1733732361538 2024-12-09T08:19:21,578 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41283:41283),(127.0.0.1/127.0.0.1:42683:42683),(127.0.0.1/127.0.0.1:42925:42925)] 2024-12-09T08:19:21,578 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ee76d2275a7974070cadc12d861d255e, NAME => 'test2727,,1733732360715.ee76d2275a7974070cadc12d861d255e.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:19:21,578 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733732360715.ee76d2275a7974070cadc12d861d255e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:21,578 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ee76d2275a7974070cadc12d861d255e 2024-12-09T08:19:21,579 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ee76d2275a7974070cadc12d861d255e 2024-12-09T08:19:21,580 INFO [StoreOpener-ee76d2275a7974070cadc12d861d255e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ee76d2275a7974070cadc12d861d255e 2024-12-09T08:19:21,581 INFO [StoreOpener-ee76d2275a7974070cadc12d861d255e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee76d2275a7974070cadc12d861d255e columnFamilyName a 2024-12-09T08:19:21,581 DEBUG [StoreOpener-ee76d2275a7974070cadc12d861d255e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:21,582 INFO [StoreOpener-ee76d2275a7974070cadc12d861d255e-1 {}] regionserver.HStore(327): Store=ee76d2275a7974070cadc12d861d255e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:21,582 INFO [StoreOpener-ee76d2275a7974070cadc12d861d255e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ee76d2275a7974070cadc12d861d255e 2024-12-09T08:19:21,583 INFO [StoreOpener-ee76d2275a7974070cadc12d861d255e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee76d2275a7974070cadc12d861d255e columnFamilyName b 2024-12-09T08:19:21,583 DEBUG [StoreOpener-ee76d2275a7974070cadc12d861d255e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:21,583 INFO [StoreOpener-ee76d2275a7974070cadc12d861d255e-1 {}] regionserver.HStore(327): Store=ee76d2275a7974070cadc12d861d255e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:21,583 INFO [StoreOpener-ee76d2275a7974070cadc12d861d255e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ee76d2275a7974070cadc12d861d255e 2024-12-09T08:19:21,584 INFO [StoreOpener-ee76d2275a7974070cadc12d861d255e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee76d2275a7974070cadc12d861d255e columnFamilyName c 2024-12-09T08:19:21,584 DEBUG [StoreOpener-ee76d2275a7974070cadc12d861d255e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:21,585 INFO [StoreOpener-ee76d2275a7974070cadc12d861d255e-1 {}] regionserver.HStore(327): Store=ee76d2275a7974070cadc12d861d255e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:21,585 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ee76d2275a7974070cadc12d861d255e 2024-12-09T08:19:21,586 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e 2024-12-09T08:19:21,588 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e 2024-12-09T08:19:21,589 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/0000000000000003000 2024-12-09T08:19:21,614 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:21,639 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/0000000000000003000 2024-12-09T08:19:21,640 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/0000000000000006000 2024-12-09T08:19:21,663 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:21,692 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/0000000000000006000 2024-12-09T08:19:21,692 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ee76d2275a7974070cadc12d861d255e 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-09T08:19:21,712 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/.tmp/a/d1d5cbf3f21740fb8d0aef71d9a94ef5 is 41, key is test2727/a:100/1733732361160/Put/seqid=0 2024-12-09T08:19:21,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741871_1048 (size=84227) 2024-12-09T08:19:21,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741871_1048 (size=84227) 2024-12-09T08:19:21,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741871_1048 (size=84227) 2024-12-09T08:19:21,718 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/.tmp/a/d1d5cbf3f21740fb8d0aef71d9a94ef5 2024-12-09T08:19:21,743 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/.tmp/b/f96d0ffd0e624f4fb0facb5e2a1630e1 is 41, key is test2727/b:100/1733732361210/Put/seqid=0 2024-12-09T08:19:21,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741872_1049 (size=84609) 2024-12-09T08:19:21,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741872_1049 (size=84609) 2024-12-09T08:19:21,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741872_1049 (size=84609) 2024-12-09T08:19:21,749 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/.tmp/b/f96d0ffd0e624f4fb0facb5e2a1630e1 2024-12-09T08:19:21,772 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/.tmp/c/522dc4af10d3439eab5d85514ac3223b is 41, key is test2727/c:100/1733732361250/Put/seqid=0 2024-12-09T08:19:21,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741873_1050 (size=84609) 2024-12-09T08:19:21,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741873_1050 (size=84609) 2024-12-09T08:19:21,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741873_1050 (size=84609) 2024-12-09T08:19:21,780 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/.tmp/c/522dc4af10d3439eab5d85514ac3223b 2024-12-09T08:19:21,788 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/.tmp/a/d1d5cbf3f21740fb8d0aef71d9a94ef5 as hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/a/d1d5cbf3f21740fb8d0aef71d9a94ef5 2024-12-09T08:19:21,795 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/a/d1d5cbf3f21740fb8d0aef71d9a94ef5, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-09T08:19:21,796 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/.tmp/b/f96d0ffd0e624f4fb0facb5e2a1630e1 as hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/b/f96d0ffd0e624f4fb0facb5e2a1630e1 2024-12-09T08:19:21,803 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/b/f96d0ffd0e624f4fb0facb5e2a1630e1, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-09T08:19:21,805 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/.tmp/c/522dc4af10d3439eab5d85514ac3223b as hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/c/522dc4af10d3439eab5d85514ac3223b 2024-12-09T08:19:21,812 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/c/522dc4af10d3439eab5d85514ac3223b, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-09T08:19:21,812 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for ee76d2275a7974070cadc12d861d255e in 120ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-09T08:19:21,813 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/0000000000000003000 2024-12-09T08:19:21,814 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/0000000000000006000 2024-12-09T08:19:21,815 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ee76d2275a7974070cadc12d861d255e 2024-12-09T08:19:21,815 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ee76d2275a7974070cadc12d861d255e 2024-12-09T08:19:21,816 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:19:21,818 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ee76d2275a7974070cadc12d861d255e 2024-12-09T08:19:21,821 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/hbase/data/default/test2727/ee76d2275a7974070cadc12d861d255e/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-09T08:19:21,822 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ee76d2275a7974070cadc12d861d255e; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62190303, jitterRate=-0.07329227030277252}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:19:21,823 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ee76d2275a7974070cadc12d861d255e: Writing region info on filesystem at 1733732361579Initializing all the Stores at 1733732361580 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732361580Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732361580Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732361580Obtaining lock to block concurrent updates at 1733732361692 (+112 ms)Preparing flush snapshotting stores in ee76d2275a7974070cadc12d861d255e at 1733732361692Finished memstore snapshotting test2727,,1733732360715.ee76d2275a7974070cadc12d861d255e., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733732361693 (+1 ms)Flushing stores of test2727,,1733732360715.ee76d2275a7974070cadc12d861d255e. at 1733732361693Flushing ee76d2275a7974070cadc12d861d255e/a: creating writer at 1733732361693Flushing ee76d2275a7974070cadc12d861d255e/a: appending metadata at 1733732361711 (+18 ms)Flushing ee76d2275a7974070cadc12d861d255e/a: closing flushed file at 1733732361711Flushing ee76d2275a7974070cadc12d861d255e/b: creating writer at 1733732361725 (+14 ms)Flushing ee76d2275a7974070cadc12d861d255e/b: appending metadata at 1733732361741 (+16 ms)Flushing ee76d2275a7974070cadc12d861d255e/b: closing flushed file at 1733732361741Flushing ee76d2275a7974070cadc12d861d255e/c: creating writer at 1733732361756 (+15 ms)Flushing ee76d2275a7974070cadc12d861d255e/c: appending metadata at 1733732361772 (+16 ms)Flushing ee76d2275a7974070cadc12d861d255e/c: closing flushed file at 1733732361772Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4151bcd2: reopening flushed file at 1733732361786 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e14cc03: reopening flushed file at 1733732361795 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@affdeeb: reopening flushed file at 1733732361803 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for ee76d2275a7974070cadc12d861d255e in 120ms, sequenceid=6000, compaction requested=false; wal=null at 1733732361812 (+9 ms)Cleaning up temporary data from old regions at 1733732361815 (+3 ms)Region opened successfully at 1733732361823 (+8 ms) 2024-12-09T08:19:21,825 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-09T08:19:21,825 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ee76d2275a7974070cadc12d861d255e, disabling compactions & flushes 2024-12-09T08:19:21,825 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733732360715.ee76d2275a7974070cadc12d861d255e. 2024-12-09T08:19:21,825 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733732360715.ee76d2275a7974070cadc12d861d255e. 2024-12-09T08:19:21,825 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733732360715.ee76d2275a7974070cadc12d861d255e. after waiting 0 ms 2024-12-09T08:19:21,825 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733732360715.ee76d2275a7974070cadc12d861d255e. 2024-12-09T08:19:21,827 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733732360715.ee76d2275a7974070cadc12d861d255e. 2024-12-09T08:19:21,827 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ee76d2275a7974070cadc12d861d255e: Waiting for close lock at 1733732361825Disabling compacts and flushes for region at 1733732361825Disabling writes for close at 1733732361825Writing region close event to WAL at 1733732361827 (+2 ms)Closed at 1733732361827 2024-12-09T08:19:21,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741870_1047 (size=175) 2024-12-09T08:19:21,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741870_1047 (size=175) 2024-12-09T08:19:21,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741870_1047 (size=175) 2024-12-09T08:19:21,833 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:19:21,833 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733732361538) 2024-12-09T08:19:21,848 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=398 (was 396) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:40400 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:55242 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:57276 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=250 (was 250), ProcessCount=11 (was 11), AvailableMemoryMB=4424 (was 4649) 2024-12-09T08:19:21,860 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=398, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=250, ProcessCount=11, AvailableMemoryMB=4423 2024-12-09T08:19:21,873 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:21,879 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733732361872, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:21,879 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733732361879 2024-12-09T08:19:21,907 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733732361872/wal.1733732361879 2024-12-09T08:19:21,909 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42683:42683),(127.0.0.1/127.0.0.1:41283:41283),(127.0.0.1/127.0.0.1:42925:42925)] 2024-12-09T08:19:21,910 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c3979179f0f9cc61ea3feecbd9f048f7, NAME => 'testSequentialEditLogSeqNum,,1733732361874.c3979179f0f9cc61ea3feecbd9f048f7.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:19:21,910 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733732361874.c3979179f0f9cc61ea3feecbd9f048f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:21,910 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c3979179f0f9cc61ea3feecbd9f048f7 2024-12-09T08:19:21,910 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c3979179f0f9cc61ea3feecbd9f048f7 2024-12-09T08:19:21,911 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:45647/hbase/data/default/testSequentialEditLogSeqNum/c3979179f0f9cc61ea3feecbd9f048f7 doesn't exist for region: c3979179f0f9cc61ea3feecbd9f048f7 on table testSequentialEditLogSeqNum 2024-12-09T08:19:21,912 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: c3979179f0f9cc61ea3feecbd9f048f7 on table testSequentialEditLogSeqNum 2024-12-09T08:19:21,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741875_1052 (size=62) 2024-12-09T08:19:21,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741875_1052 (size=62) 2024-12-09T08:19:21,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741875_1052 (size=62) 2024-12-09T08:19:21,925 INFO [StoreOpener-c3979179f0f9cc61ea3feecbd9f048f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c3979179f0f9cc61ea3feecbd9f048f7 2024-12-09T08:19:21,927 INFO [StoreOpener-c3979179f0f9cc61ea3feecbd9f048f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3979179f0f9cc61ea3feecbd9f048f7 columnFamilyName a 2024-12-09T08:19:21,927 DEBUG [StoreOpener-c3979179f0f9cc61ea3feecbd9f048f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:21,928 INFO [StoreOpener-c3979179f0f9cc61ea3feecbd9f048f7-1 {}] regionserver.HStore(327): Store=c3979179f0f9cc61ea3feecbd9f048f7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:21,928 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c3979179f0f9cc61ea3feecbd9f048f7 2024-12-09T08:19:21,929 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testSequentialEditLogSeqNum/c3979179f0f9cc61ea3feecbd9f048f7 2024-12-09T08:19:21,929 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testSequentialEditLogSeqNum/c3979179f0f9cc61ea3feecbd9f048f7 2024-12-09T08:19:21,929 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c3979179f0f9cc61ea3feecbd9f048f7 2024-12-09T08:19:21,929 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c3979179f0f9cc61ea3feecbd9f048f7 2024-12-09T08:19:21,931 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c3979179f0f9cc61ea3feecbd9f048f7 2024-12-09T08:19:21,933 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/hbase/data/default/testSequentialEditLogSeqNum/c3979179f0f9cc61ea3feecbd9f048f7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:19:21,934 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c3979179f0f9cc61ea3feecbd9f048f7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67610040, jitterRate=0.007468104362487793}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T08:19:21,934 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c3979179f0f9cc61ea3feecbd9f048f7: Writing region info on filesystem at 1733732361911Initializing all the Stores at 1733732361925 (+14 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732361925Cleaning up temporary data from old regions at 1733732361929 (+4 ms)Region opened successfully at 1733732361934 (+5 ms) 2024-12-09T08:19:21,945 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c3979179f0f9cc61ea3feecbd9f048f7 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-09T08:19:21,963 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testSequentialEditLogSeqNum/c3979179f0f9cc61ea3feecbd9f048f7/.tmp/a/48f1a1d34cfc471590ced624595c391a is 81, key is testSequentialEditLogSeqNum/a:x0/1733732361934/Put/seqid=0 2024-12-09T08:19:21,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741876_1053 (size=5833) 2024-12-09T08:19:21,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741876_1053 (size=5833) 2024-12-09T08:19:21,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741876_1053 (size=5833) 2024-12-09T08:19:21,970 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/testSequentialEditLogSeqNum/c3979179f0f9cc61ea3feecbd9f048f7/.tmp/a/48f1a1d34cfc471590ced624595c391a 2024-12-09T08:19:21,977 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testSequentialEditLogSeqNum/c3979179f0f9cc61ea3feecbd9f048f7/.tmp/a/48f1a1d34cfc471590ced624595c391a as hdfs://localhost:45647/hbase/data/default/testSequentialEditLogSeqNum/c3979179f0f9cc61ea3feecbd9f048f7/a/48f1a1d34cfc471590ced624595c391a 2024-12-09T08:19:21,983 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/testSequentialEditLogSeqNum/c3979179f0f9cc61ea3feecbd9f048f7/a/48f1a1d34cfc471590ced624595c391a, entries=10, sequenceid=13, filesize=5.7 K 2024-12-09T08:19:21,984 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for c3979179f0f9cc61ea3feecbd9f048f7 in 39ms, sequenceid=13, compaction requested=false 2024-12-09T08:19:21,985 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c3979179f0f9cc61ea3feecbd9f048f7: 2024-12-09T08:19:21,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:19:21,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:19:21,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:19:21,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:19:21,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T08:19:21,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741874_1051 (size=3256) 2024-12-09T08:19:21,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741874_1051 (size=3256) 2024-12-09T08:19:21,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741874_1051 (size=3256) 2024-12-09T08:19:22,010 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45647/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733732361872/wal.1733732361879, size=3.2 K (3256bytes) 2024-12-09T08:19:22,010 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45647/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733732361872/wal.1733732361879 2024-12-09T08:19:22,011 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45647/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733732361872/wal.1733732361879 after 1ms 2024-12-09T08:19:22,037 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:22,037 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45647/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733732361872/wal.1733732361879 took 27ms 2024-12-09T08:19:22,040 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45647/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733732361872/wal.1733732361879 so closing down 2024-12-09T08:19:22,040 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:19:22,041 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733732361879.temp 2024-12-09T08:19:22,065 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testSequentialEditLogSeqNum/c3979179f0f9cc61ea3feecbd9f048f7/recovered.edits/0000000000000000003-wal.1733732361879.temp 2024-12-09T08:19:22,065 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:19:22,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741877_1054 (size=2650) 2024-12-09T08:19:22,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741877_1054 (size=2650) 2024-12-09T08:19:22,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741877_1054 (size=2650) 2024-12-09T08:19:22,075 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testSequentialEditLogSeqNum/c3979179f0f9cc61ea3feecbd9f048f7/recovered.edits/0000000000000000003-wal.1733732361879.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-09T08:19:22,076 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45647/hbase/data/default/testSequentialEditLogSeqNum/c3979179f0f9cc61ea3feecbd9f048f7/recovered.edits/0000000000000000003-wal.1733732361879.temp to hdfs://localhost:45647/hbase/data/default/testSequentialEditLogSeqNum/c3979179f0f9cc61ea3feecbd9f048f7/recovered.edits/0000000000000000020 2024-12-09T08:19:22,076 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 38 ms; skipped=2; WAL=hdfs://localhost:45647/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733732361872/wal.1733732361879, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-12-09T08:19:22,076 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45647/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733732361872/wal.1733732361879, journal: Splitting hdfs://localhost:45647/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733732361872/wal.1733732361879, size=3.2 K (3256bytes) at 1733732362010Finishing writing output for hdfs://localhost:45647/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733732361872/wal.1733732361879 so closing down at 1733732362040 (+30 ms)Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testSequentialEditLogSeqNum/c3979179f0f9cc61ea3feecbd9f048f7/recovered.edits/0000000000000000003-wal.1733732361879.temp at 1733732362065 (+25 ms)3 split writer threads finished at 1733732362065Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testSequentialEditLogSeqNum/c3979179f0f9cc61ea3feecbd9f048f7/recovered.edits/0000000000000000003-wal.1733732361879.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733732362075 (+10 ms)Rename recovered edits hdfs://localhost:45647/hbase/data/default/testSequentialEditLogSeqNum/c3979179f0f9cc61ea3feecbd9f048f7/recovered.edits/0000000000000000003-wal.1733732361879.temp to hdfs://localhost:45647/hbase/data/default/testSequentialEditLogSeqNum/c3979179f0f9cc61ea3feecbd9f048f7/recovered.edits/0000000000000000020 at 1733732362076 (+1 ms)Processed 17 edits across 1 Regions in 38 ms; skipped=2; WAL=hdfs://localhost:45647/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733732361872/wal.1733732361879, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1733732362076 2024-12-09T08:19:22,088 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=405 (was 398) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:40400 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/cluster_95cc11f3-affe-6e07-bfcd-38f5721bf4f5/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:57276 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/cluster_95cc11f3-affe-6e07-bfcd-38f5721bf4f5/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=250 (was 250), ProcessCount=11 (was 11), AvailableMemoryMB=4421 (was 4423) 2024-12-09T08:19:22,099 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=405, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=250, ProcessCount=11, AvailableMemoryMB=4420 2024-12-09T08:19:22,115 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:22,117 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:22,141 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:19:22,145 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-72921317, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/hregion-72921317, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:22,162 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-72921317/hregion-72921317.1733732362146, exclude list is [], retry=0 2024-12-09T08:19:22,165 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:22,165 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:22,165 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:22,187 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-72921317/hregion-72921317.1733732362146 2024-12-09T08:19:22,188 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42683:42683),(127.0.0.1/127.0.0.1:41283:41283),(127.0.0.1/127.0.0.1:42925:42925)] 2024-12-09T08:19:22,188 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 25698ed88667c5aefaff23ffa972afe3, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733732362116.25698ed88667c5aefaff23ffa972afe3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45647/hbase 2024-12-09T08:19:22,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741879_1056 (size=70) 2024-12-09T08:19:22,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741879_1056 (size=70) 2024-12-09T08:19:22,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741879_1056 (size=70) 2024-12-09T08:19:22,197 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733732362116.25698ed88667c5aefaff23ffa972afe3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:22,198 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:22,200 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25698ed88667c5aefaff23ffa972afe3 columnFamilyName a 2024-12-09T08:19:22,200 DEBUG [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:22,200 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] regionserver.HStore(327): Store=25698ed88667c5aefaff23ffa972afe3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:22,200 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:22,202 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25698ed88667c5aefaff23ffa972afe3 columnFamilyName b 2024-12-09T08:19:22,202 DEBUG [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:22,203 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] regionserver.HStore(327): Store=25698ed88667c5aefaff23ffa972afe3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:22,203 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:22,204 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25698ed88667c5aefaff23ffa972afe3 columnFamilyName c 2024-12-09T08:19:22,204 DEBUG [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:22,205 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] regionserver.HStore(327): Store=25698ed88667c5aefaff23ffa972afe3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:22,205 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:22,206 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:22,206 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:22,208 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:22,208 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:22,208 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:19:22,210 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:22,212 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:19:22,213 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 25698ed88667c5aefaff23ffa972afe3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61960976, jitterRate=-0.07670950889587402}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:19:22,215 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 25698ed88667c5aefaff23ffa972afe3: Writing region info on filesystem at 1733732362197Initializing all the Stores at 1733732362198 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732362198Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732362198Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732362198Cleaning up temporary data from old regions at 1733732362208 (+10 ms)Region opened successfully at 1733732362214 (+6 ms) 2024-12-09T08:19:22,215 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 25698ed88667c5aefaff23ffa972afe3, disabling compactions & flushes 2024-12-09T08:19:22,215 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733732362116.25698ed88667c5aefaff23ffa972afe3. 2024-12-09T08:19:22,215 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733732362116.25698ed88667c5aefaff23ffa972afe3. 2024-12-09T08:19:22,215 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733732362116.25698ed88667c5aefaff23ffa972afe3. after waiting 0 ms 2024-12-09T08:19:22,215 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733732362116.25698ed88667c5aefaff23ffa972afe3. 2024-12-09T08:19:22,215 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733732362116.25698ed88667c5aefaff23ffa972afe3. 2024-12-09T08:19:22,216 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 25698ed88667c5aefaff23ffa972afe3: Waiting for close lock at 1733732362215Disabling compacts and flushes for region at 1733732362215Disabling writes for close at 1733732362215Writing region close event to WAL at 1733732362215Closed at 1733732362215 2024-12-09T08:19:22,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741878_1055 (size=175) 2024-12-09T08:19:22,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741878_1055 (size=175) 2024-12-09T08:19:22,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741878_1055 (size=175) 2024-12-09T08:19:22,222 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:19:22,223 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-72921317:(num 1733732362146) 2024-12-09T08:19:22,223 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:19:22,225 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732362114, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:22,238 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732362114/wal.1733732362226, exclude list is [], retry=0 2024-12-09T08:19:22,241 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:22,242 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:22,242 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:22,265 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732362114/wal.1733732362226 2024-12-09T08:19:22,266 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41283:41283),(127.0.0.1/127.0.0.1:42683:42683),(127.0.0.1/127.0.0.1:42925:42925)] 2024-12-09T08:19:22,266 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 25698ed88667c5aefaff23ffa972afe3, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733732362116.25698ed88667c5aefaff23ffa972afe3.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:19:22,266 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733732362116.25698ed88667c5aefaff23ffa972afe3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:22,266 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:22,266 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:22,267 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:22,268 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25698ed88667c5aefaff23ffa972afe3 columnFamilyName a 2024-12-09T08:19:22,268 DEBUG [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:22,269 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] regionserver.HStore(327): Store=25698ed88667c5aefaff23ffa972afe3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:22,269 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:22,270 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25698ed88667c5aefaff23ffa972afe3 columnFamilyName b 2024-12-09T08:19:22,270 DEBUG [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:22,271 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] regionserver.HStore(327): Store=25698ed88667c5aefaff23ffa972afe3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:22,271 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:22,271 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25698ed88667c5aefaff23ffa972afe3 columnFamilyName c 2024-12-09T08:19:22,271 DEBUG [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:22,272 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] regionserver.HStore(327): Store=25698ed88667c5aefaff23ffa972afe3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:22,272 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:22,273 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:22,274 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:22,276 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:22,276 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:22,276 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:19:22,278 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:22,279 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 25698ed88667c5aefaff23ffa972afe3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66438283, jitterRate=-0.009992435574531555}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:19:22,280 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 25698ed88667c5aefaff23ffa972afe3: Writing region info on filesystem at 1733732362266Initializing all the Stores at 1733732362267 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732362267Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732362267Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732362267Cleaning up temporary data from old regions at 1733732362276 (+9 ms)Region opened successfully at 1733732362280 (+4 ms) 2024-12-09T08:19:22,284 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733732362283/Put/seqid=0 2024-12-09T08:19:22,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741881_1058 (size=4826) 2024-12-09T08:19:22,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741881_1058 (size=4826) 2024-12-09T08:19:22,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741881_1058 (size=4826) 2024-12-09T08:19:22,291 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45647/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 25698ed88667c5aefaff23ffa972afe3/a 2024-12-09T08:19:22,298 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-09T08:19:22,298 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T08:19:22,298 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 25698ed88667c5aefaff23ffa972afe3: 2024-12-09T08:19:22,300 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3/a/8494f2c298804e4fa0e2fedc36a193ff_SeqId_3_ 2024-12-09T08:19:22,301 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45647/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 25698ed88667c5aefaff23ffa972afe3/a as hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3/a/8494f2c298804e4fa0e2fedc36a193ff_SeqId_3_ - updating store file list. 2024-12-09T08:19:22,307 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 8494f2c298804e4fa0e2fedc36a193ff_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-09T08:19:22,307 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3/a/8494f2c298804e4fa0e2fedc36a193ff_SeqId_3_ into 25698ed88667c5aefaff23ffa972afe3/a 2024-12-09T08:19:22,307 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45647/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 25698ed88667c5aefaff23ffa972afe3/a (new location: hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3/a/8494f2c298804e4fa0e2fedc36a193ff_SeqId_3_) 2024-12-09T08:19:22,347 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732362114/wal.1733732362226, size=0 (0bytes) 2024-12-09T08:19:22,347 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:45647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732362114/wal.1733732362226 might be still open, length is 0 2024-12-09T08:19:22,347 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732362114/wal.1733732362226 2024-12-09T08:19:22,348 WARN [IPC Server handler 0 on default port 45647 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732362114/wal.1733732362226 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-09T08:19:22,348 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732362114/wal.1733732362226 after 1ms 2024-12-09T08:19:24,926 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:40498 [Receiving block BP-1712704723-172.17.0.2-1733732346335:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:45463:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40498 dst: /127.0.0.1:45463 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45463 remote=/127.0.0.1:40498]. Total timeout mills is 60000, 57387 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:24,927 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:55378 [Receiving block BP-1712704723-172.17.0.2-1733732346335:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:41399:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55378 dst: /127.0.0.1:41399 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:24,928 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:57348 [Receiving block BP-1712704723-172.17.0.2-1733732346335:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:38689:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57348 dst: /127.0.0.1:38689 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:24,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741880_1059 (size=606) 2024-12-09T08:19:24,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741880_1059 (size=606) 2024-12-09T08:19:26,350 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732362114/wal.1733732362226 after 4003ms 2024-12-09T08:19:26,378 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:26,378 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732362114/wal.1733732362226 took 4031ms 2024-12-09T08:19:26,380 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:45647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732362114/wal.1733732362226; continuing. 2024-12-09T08:19:26,380 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732362114/wal.1733732362226 so closing down 2024-12-09T08:19:26,380 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:19:26,382 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733732362226.temp 2024-12-09T08:19:26,405 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3/recovered.edits/0000000000000000005-wal.1733732362226.temp 2024-12-09T08:19:26,406 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:19:26,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741882_1060 (size=329) 2024-12-09T08:19:26,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741882_1060 (size=329) 2024-12-09T08:19:26,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741882_1060 (size=329) 2024-12-09T08:19:26,413 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3/recovered.edits/0000000000000000005-wal.1733732362226.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-09T08:19:26,414 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3/recovered.edits/0000000000000000005-wal.1733732362226.temp to hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3/recovered.edits/0000000000000000005 2024-12-09T08:19:26,414 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:45647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732362114/wal.1733732362226, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T08:19:26,414 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732362114/wal.1733732362226, journal: Splitting hdfs://localhost:45647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732362114/wal.1733732362226, size=0 (0bytes) at 1733732362347Finishing writing output for hdfs://localhost:45647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732362114/wal.1733732362226 so closing down at 1733732366380 (+4033 ms)Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3/recovered.edits/0000000000000000005-wal.1733732362226.temp at 1733732366406 (+26 ms)3 split writer threads finished at 1733732366406Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3/recovered.edits/0000000000000000005-wal.1733732362226.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733732366413 (+7 ms)Rename recovered edits hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3/recovered.edits/0000000000000000005-wal.1733732362226.temp to hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3/recovered.edits/0000000000000000005 at 1733732366414 (+1 ms)Processed 2 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:45647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732362114/wal.1733732362226, size=0, length=0, corrupted=false, cancelled=false at 1733732366414 2024-12-09T08:19:26,416 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732362114/wal.1733732362226 to hdfs://localhost:45647/hbase/oldWALs/wal.1733732362226 2024-12-09T08:19:26,416 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3/recovered.edits/0000000000000000005 2024-12-09T08:19:26,417 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:19:26,418 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732362114, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:26,430 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732362114/wal.1733732366418, exclude list is [], retry=0 2024-12-09T08:19:26,433 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:26,433 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:26,433 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:26,456 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732362114/wal.1733732366418 2024-12-09T08:19:26,457 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41283:41283),(127.0.0.1/127.0.0.1:42925:42925),(127.0.0.1/127.0.0.1:42683:42683)] 2024-12-09T08:19:26,457 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 25698ed88667c5aefaff23ffa972afe3, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733732362116.25698ed88667c5aefaff23ffa972afe3.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:19:26,457 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733732362116.25698ed88667c5aefaff23ffa972afe3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:26,457 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:26,457 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:26,459 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:26,460 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25698ed88667c5aefaff23ffa972afe3 columnFamilyName a 2024-12-09T08:19:26,460 DEBUG [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:26,466 DEBUG [StoreFileOpener-25698ed88667c5aefaff23ffa972afe3-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 8494f2c298804e4fa0e2fedc36a193ff_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-09T08:19:26,466 DEBUG [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3/a/8494f2c298804e4fa0e2fedc36a193ff_SeqId_3_ 2024-12-09T08:19:26,467 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] regionserver.HStore(327): Store=25698ed88667c5aefaff23ffa972afe3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:26,467 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:26,468 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25698ed88667c5aefaff23ffa972afe3 columnFamilyName b 2024-12-09T08:19:26,468 DEBUG [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:26,468 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] regionserver.HStore(327): Store=25698ed88667c5aefaff23ffa972afe3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:26,469 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:26,470 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25698ed88667c5aefaff23ffa972afe3 columnFamilyName c 2024-12-09T08:19:26,470 DEBUG [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:26,470 INFO [StoreOpener-25698ed88667c5aefaff23ffa972afe3-1 {}] regionserver.HStore(327): Store=25698ed88667c5aefaff23ffa972afe3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:26,470 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:26,471 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:26,473 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:26,474 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3/recovered.edits/0000000000000000005 2024-12-09T08:19:26,498 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:26,499 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3/recovered.edits/0000000000000000005 2024-12-09T08:19:26,499 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 25698ed88667c5aefaff23ffa972afe3 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-09T08:19:26,513 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3/.tmp/a/59638142f1814048a307dc72ecc9d6fe is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733732362313/Put/seqid=0 2024-12-09T08:19:26,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741884_1062 (size=5149) 2024-12-09T08:19:26,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741884_1062 (size=5149) 2024-12-09T08:19:26,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741884_1062 (size=5149) 2024-12-09T08:19:26,520 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3/.tmp/a/59638142f1814048a307dc72ecc9d6fe 2024-12-09T08:19:26,527 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3/.tmp/a/59638142f1814048a307dc72ecc9d6fe as hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3/a/59638142f1814048a307dc72ecc9d6fe 2024-12-09T08:19:26,533 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3/a/59638142f1814048a307dc72ecc9d6fe, entries=1, sequenceid=5, filesize=5.0 K 2024-12-09T08:19:26,534 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 25698ed88667c5aefaff23ffa972afe3 in 34ms, sequenceid=5, compaction requested=false; wal=null 2024-12-09T08:19:26,534 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3/recovered.edits/0000000000000000005 2024-12-09T08:19:26,536 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:26,536 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:26,536 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:19:26,538 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 25698ed88667c5aefaff23ffa972afe3 2024-12-09T08:19:26,541 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/25698ed88667c5aefaff23ffa972afe3/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-09T08:19:26,542 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 25698ed88667c5aefaff23ffa972afe3; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66836021, jitterRate=-0.004065677523612976}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:19:26,542 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 25698ed88667c5aefaff23ffa972afe3: Writing region info on filesystem at 1733732366458Initializing all the Stores at 1733732366459 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732366459Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732366459Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732366459Obtaining lock to block concurrent updates at 1733732366499 (+40 ms)Preparing flush snapshotting stores in 25698ed88667c5aefaff23ffa972afe3 at 1733732366499Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733732362116.25698ed88667c5aefaff23ffa972afe3., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733732366499Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733732362116.25698ed88667c5aefaff23ffa972afe3. at 1733732366499Flushing 25698ed88667c5aefaff23ffa972afe3/a: creating writer at 1733732366499Flushing 25698ed88667c5aefaff23ffa972afe3/a: appending metadata at 1733732366513 (+14 ms)Flushing 25698ed88667c5aefaff23ffa972afe3/a: closing flushed file at 1733732366513Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@134a2e9c: reopening flushed file at 1733732366525 (+12 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 25698ed88667c5aefaff23ffa972afe3 in 34ms, sequenceid=5, compaction requested=false; wal=null at 1733732366534 (+9 ms)Cleaning up temporary data from old regions at 1733732366536 (+2 ms)Region opened successfully at 1733732366542 (+6 ms) 2024-12-09T08:19:26,547 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 25698ed88667c5aefaff23ffa972afe3, disabling compactions & flushes 2024-12-09T08:19:26,547 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733732362116.25698ed88667c5aefaff23ffa972afe3. 2024-12-09T08:19:26,547 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733732362116.25698ed88667c5aefaff23ffa972afe3. 2024-12-09T08:19:26,547 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733732362116.25698ed88667c5aefaff23ffa972afe3. after waiting 0 ms 2024-12-09T08:19:26,547 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733732362116.25698ed88667c5aefaff23ffa972afe3. 2024-12-09T08:19:26,548 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733732362116.25698ed88667c5aefaff23ffa972afe3. 2024-12-09T08:19:26,548 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 25698ed88667c5aefaff23ffa972afe3: Waiting for close lock at 1733732366547Disabling compacts and flushes for region at 1733732366547Disabling writes for close at 1733732366547Writing region close event to WAL at 1733732366548 (+1 ms)Closed at 1733732366548 2024-12-09T08:19:26,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741883_1061 (size=175) 2024-12-09T08:19:26,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741883_1061 (size=175) 2024-12-09T08:19:26,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741883_1061 (size=175) 2024-12-09T08:19:26,554 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:19:26,554 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733732366418) 2024-12-09T08:19:26,566 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=407 (was 405) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1027215390_22 at /127.0.0.1:40508 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:45647 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1027215390_22 at /127.0.0.1:55428 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:45647 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1022 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=237 (was 250), ProcessCount=11 (was 11), AvailableMemoryMB=4406 (was 4420) 2024-12-09T08:19:26,579 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=407, OpenFileDescriptor=1022, MaxFileDescriptor=1048576, SystemLoadAverage=237, ProcessCount=11, AvailableMemoryMB=4405 2024-12-09T08:19:26,593 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:26,597 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T08:19:26,600 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 5bda946059d1,41647,1733732350732 2024-12-09T08:19:26,603 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@f3d14a6 2024-12-09T08:19:26,605 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T08:19:26,607 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T08:19:26,611 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T08:19:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-09T08:19:26,618 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T08:19:26,620 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-09T08:19:26,620 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:26,622 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T08:19:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T08:19:26,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741885_1063 (size=694) 2024-12-09T08:19:26,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741885_1063 (size=694) 2024-12-09T08:19:26,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741885_1063 (size=694) 2024-12-09T08:19:26,634 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7a8f9beb16114c9e9ebf1936c0e75d39, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3 2024-12-09T08:19:26,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741886_1064 (size=77) 2024-12-09T08:19:26,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741886_1064 (size=77) 2024-12-09T08:19:26,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741886_1064 (size=77) 2024-12-09T08:19:26,642 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:26,642 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 7a8f9beb16114c9e9ebf1936c0e75d39, disabling compactions & flushes 2024-12-09T08:19:26,642 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:26,642 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:26,643 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. after waiting 0 ms 2024-12-09T08:19:26,643 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:26,643 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:26,643 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7a8f9beb16114c9e9ebf1936c0e75d39: Waiting for close lock at 1733732366642Disabling compacts and flushes for region at 1733732366642Disabling writes for close at 1733732366643 (+1 ms)Writing region close event to WAL at 1733732366643Closed at 1733732366643 2024-12-09T08:19:26,644 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T08:19:26,648 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733732366644"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733732366644"}]},"ts":"1733732366644"} 2024-12-09T08:19:26,651 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T08:19:26,652 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T08:19:26,655 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732366652"}]},"ts":"1733732366652"} 2024-12-09T08:19:26,658 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-09T08:19:26,659 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {5bda946059d1=0} racks are {/default-rack=0} 2024-12-09T08:19:26,660 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T08:19:26,660 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T08:19:26,660 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-09T08:19:26,660 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T08:19:26,660 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T08:19:26,660 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-09T08:19:26,660 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T08:19:26,660 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T08:19:26,660 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-09T08:19:26,660 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T08:19:26,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7a8f9beb16114c9e9ebf1936c0e75d39, ASSIGN}] 2024-12-09T08:19:26,663 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7a8f9beb16114c9e9ebf1936c0e75d39, ASSIGN 2024-12-09T08:19:26,665 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7a8f9beb16114c9e9ebf1936c0e75d39, ASSIGN; state=OFFLINE, location=5bda946059d1,33313,1733732351482; forceNewPlan=false, retain=false 2024-12-09T08:19:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T08:19:26,820 INFO [5bda946059d1:41647 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T08:19:26,822 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7a8f9beb16114c9e9ebf1936c0e75d39, regionState=OPENING, regionLocation=5bda946059d1,33313,1733732351482 2024-12-09T08:19:26,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7a8f9beb16114c9e9ebf1936c0e75d39, ASSIGN because future has completed 2024-12-09T08:19:26,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7a8f9beb16114c9e9ebf1936c0e75d39, server=5bda946059d1,33313,1733732351482}] 2024-12-09T08:19:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T08:19:26,981 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T08:19:26,983 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59931, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T08:19:26,990 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:26,990 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 7a8f9beb16114c9e9ebf1936c0e75d39, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:19:26,991 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:26,991 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:26,991 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:26,991 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:26,994 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:26,997 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a8f9beb16114c9e9ebf1936c0e75d39 columnFamilyName cf1 2024-12-09T08:19:26,997 DEBUG [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:26,997 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.HStore(327): Store=7a8f9beb16114c9e9ebf1936c0e75d39/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:26,997 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:26,999 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a8f9beb16114c9e9ebf1936c0e75d39 columnFamilyName cf2 2024-12-09T08:19:26,999 DEBUG [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:27,000 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.HStore(327): Store=7a8f9beb16114c9e9ebf1936c0e75d39/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:27,000 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:27,001 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:27,002 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:27,002 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:27,003 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:27,004 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T08:19:27,005 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:27,008 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:19:27,008 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 7a8f9beb16114c9e9ebf1936c0e75d39; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73293230, jitterRate=0.09215423464775085}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T08:19:27,009 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:27,009 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 7a8f9beb16114c9e9ebf1936c0e75d39: Running coprocessor pre-open hook at 1733732366991Writing region info on filesystem at 1733732366992 (+1 ms)Initializing all the Stores at 1733732366993 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732366993Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732366994 (+1 ms)Cleaning up temporary data from old regions at 1733732367003 (+9 ms)Running coprocessor post-open hooks at 1733732367009 (+6 ms)Region opened successfully at 1733732367009 2024-12-09T08:19:27,011 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., pid=6, masterSystemTime=1733732366981 2024-12-09T08:19:27,013 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:27,013 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:27,014 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7a8f9beb16114c9e9ebf1936c0e75d39, regionState=OPEN, openSeqNum=2, regionLocation=5bda946059d1,33313,1733732351482 2024-12-09T08:19:27,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7a8f9beb16114c9e9ebf1936c0e75d39, server=5bda946059d1,33313,1733732351482 because future has completed 2024-12-09T08:19:27,022 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-09T08:19:27,022 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7a8f9beb16114c9e9ebf1936c0e75d39, server=5bda946059d1,33313,1733732351482 in 191 msec 2024-12-09T08:19:27,025 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-09T08:19:27,025 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7a8f9beb16114c9e9ebf1936c0e75d39, ASSIGN in 360 msec 2024-12-09T08:19:27,026 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T08:19:27,026 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733732367026"}]},"ts":"1733732367026"} 2024-12-09T08:19:27,029 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-09T08:19:27,031 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T08:19:27,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 419 msec 2024-12-09T08:19:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T08:19:27,257 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-09T08:19:27,257 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-09T08:19:27,259 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T08:19:27,267 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-09T08:19:27,268 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T08:19:27,268 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-09T08:19:27,281 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,33313,1733732351482, seqNum=2] 2024-12-09T08:19:27,282 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:19:27,284 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:19:27,299 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=7a8f9beb16114c9e9ebf1936c0e75d39, source=5bda946059d1,33313,1733732351482, destination=5bda946059d1,34083,1733732351576, warming up region on 5bda946059d1,34083,1733732351576 2024-12-09T08:19:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T08:19:27,302 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=7a8f9beb16114c9e9ebf1936c0e75d39, source=5bda946059d1,33313,1733732351482, destination=5bda946059d1,34083,1733732351576, running balancer 2024-12-09T08:19:27,303 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49153, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T08:19:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7a8f9beb16114c9e9ebf1936c0e75d39, REOPEN/MOVE 2024-12-09T08:19:27,304 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7a8f9beb16114c9e9ebf1936c0e75d39, REOPEN/MOVE 2024-12-09T08:19:27,306 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7a8f9beb16114c9e9ebf1936c0e75d39, regionState=CLOSING, regionLocation=5bda946059d1,33313,1733732351482 2024-12-09T08:19:27,309 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34083 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34083 {}] regionserver.HRegion(7855): Warmup {ENCODED => 7a8f9beb16114c9e9ebf1936c0e75d39, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:19:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34083 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:27,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7a8f9beb16114c9e9ebf1936c0e75d39, REOPEN/MOVE because future has completed 2024-12-09T08:19:27,310 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:27,310 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T08:19:27,311 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7a8f9beb16114c9e9ebf1936c0e75d39, server=5bda946059d1,33313,1733732351482}] 2024-12-09T08:19:27,311 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a8f9beb16114c9e9ebf1936c0e75d39 columnFamilyName cf1 2024-12-09T08:19:27,311 DEBUG [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:27,311 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.HStore(327): Store=7a8f9beb16114c9e9ebf1936c0e75d39/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:27,311 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:27,312 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a8f9beb16114c9e9ebf1936c0e75d39 columnFamilyName cf2 2024-12-09T08:19:27,312 DEBUG [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:27,313 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.HStore(327): Store=7a8f9beb16114c9e9ebf1936c0e75d39/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34083 {}] regionserver.HRegion(1722): Closing 7a8f9beb16114c9e9ebf1936c0e75d39, disabling compactions & flushes 2024-12-09T08:19:27,313 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34083 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34083 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34083 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. after waiting 0 ms 2024-12-09T08:19:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34083 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:27,314 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34083 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34083 {}] regionserver.HRegion(1676): Region close journal for 7a8f9beb16114c9e9ebf1936c0e75d39: Waiting for close lock at 1733732367313Disabling compacts and flushes for region at 1733732367313Disabling writes for close at 1733732367313Writing region close event to WAL at 1733732367314 (+1 ms)Closed at 1733732367314 2024-12-09T08:19:27,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-09T08:19:27,469 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:27,469 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T08:19:27,469 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 7a8f9beb16114c9e9ebf1936c0e75d39, disabling compactions & flushes 2024-12-09T08:19:27,469 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:27,470 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:27,470 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. after waiting 0 ms 2024-12-09T08:19:27,470 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:27,470 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 7a8f9beb16114c9e9ebf1936c0e75d39 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-09T08:19:27,486 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/.tmp/cf1/2129f1563c1e4efe9ea6022bea37442e is 35, key is r1/cf1:q/1733732367285/Put/seqid=0 2024-12-09T08:19:27,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741887_1065 (size=4783) 2024-12-09T08:19:27,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741887_1065 (size=4783) 2024-12-09T08:19:27,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741887_1065 (size=4783) 2024-12-09T08:19:27,496 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/.tmp/cf1/2129f1563c1e4efe9ea6022bea37442e 2024-12-09T08:19:27,504 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/.tmp/cf1/2129f1563c1e4efe9ea6022bea37442e as hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf1/2129f1563c1e4efe9ea6022bea37442e 2024-12-09T08:19:27,512 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf1/2129f1563c1e4efe9ea6022bea37442e, entries=1, sequenceid=5, filesize=4.7 K 2024-12-09T08:19:27,513 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 7a8f9beb16114c9e9ebf1936c0e75d39 in 43ms, sequenceid=5, compaction requested=false 2024-12-09T08:19:27,513 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-09T08:19:27,519 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-09T08:19:27,522 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:27,522 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 7a8f9beb16114c9e9ebf1936c0e75d39: Waiting for close lock at 1733732367469Running coprocessor pre-close hooks at 1733732367469Disabling compacts and flushes for region at 1733732367469Disabling writes for close at 1733732367470 (+1 ms)Obtaining lock to block concurrent updates at 1733732367470Preparing flush snapshotting stores in 7a8f9beb16114c9e9ebf1936c0e75d39 at 1733732367470Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733732367470Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. at 1733732367471 (+1 ms)Flushing 7a8f9beb16114c9e9ebf1936c0e75d39/cf1: creating writer at 1733732367471Flushing 7a8f9beb16114c9e9ebf1936c0e75d39/cf1: appending metadata at 1733732367486 (+15 ms)Flushing 7a8f9beb16114c9e9ebf1936c0e75d39/cf1: closing flushed file at 1733732367486Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e14e18f: reopening flushed file at 1733732367503 (+17 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 7a8f9beb16114c9e9ebf1936c0e75d39 in 43ms, sequenceid=5, compaction requested=false at 1733732367513 (+10 ms)Writing region close event to WAL at 1733732367515 (+2 ms)Running coprocessor post-close hooks at 1733732367520 (+5 ms)Closed at 1733732367521 (+1 ms) 2024-12-09T08:19:27,522 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 7a8f9beb16114c9e9ebf1936c0e75d39 move to 5bda946059d1,34083,1733732351576 record at close sequenceid=5 2024-12-09T08:19:27,525 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:27,526 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7a8f9beb16114c9e9ebf1936c0e75d39, regionState=CLOSED 2024-12-09T08:19:27,529 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7a8f9beb16114c9e9ebf1936c0e75d39, server=5bda946059d1,33313,1733732351482 because future has completed 2024-12-09T08:19:27,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-09T08:19:27,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 7a8f9beb16114c9e9ebf1936c0e75d39, server=5bda946059d1,33313,1733732351482 in 220 msec 2024-12-09T08:19:27,537 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7a8f9beb16114c9e9ebf1936c0e75d39, REOPEN/MOVE; state=CLOSED, location=5bda946059d1,34083,1733732351576; forceNewPlan=false, retain=false 2024-12-09T08:19:27,687 INFO [5bda946059d1:41647 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T08:19:27,687 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7a8f9beb16114c9e9ebf1936c0e75d39, regionState=OPENING, regionLocation=5bda946059d1,34083,1733732351576 2024-12-09T08:19:27,692 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7a8f9beb16114c9e9ebf1936c0e75d39, REOPEN/MOVE because future has completed 2024-12-09T08:19:27,692 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7a8f9beb16114c9e9ebf1936c0e75d39, server=5bda946059d1,34083,1733732351576}] 2024-12-09T08:19:27,856 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:27,856 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 7a8f9beb16114c9e9ebf1936c0e75d39, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:19:27,857 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:27,857 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:27,857 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:27,857 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:27,859 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:27,861 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a8f9beb16114c9e9ebf1936c0e75d39 columnFamilyName cf1 2024-12-09T08:19:27,861 DEBUG [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:27,867 DEBUG [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf1/2129f1563c1e4efe9ea6022bea37442e 2024-12-09T08:19:27,867 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.HStore(327): Store=7a8f9beb16114c9e9ebf1936c0e75d39/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:27,867 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:27,868 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a8f9beb16114c9e9ebf1936c0e75d39 columnFamilyName cf2 2024-12-09T08:19:27,868 DEBUG [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:27,869 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.HStore(327): Store=7a8f9beb16114c9e9ebf1936c0e75d39/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:27,869 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:27,870 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:27,871 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:27,872 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:27,872 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:27,873 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T08:19:27,874 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:27,875 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 7a8f9beb16114c9e9ebf1936c0e75d39; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65307855, jitterRate=-0.026837125420570374}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T08:19:27,875 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:27,875 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 7a8f9beb16114c9e9ebf1936c0e75d39: Running coprocessor pre-open hook at 1733732367857Writing region info on filesystem at 1733732367857Initializing all the Stores at 1733732367858 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732367858Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732367859 (+1 ms)Cleaning up temporary data from old regions at 1733732367872 (+13 ms)Running coprocessor post-open hooks at 1733732367875 (+3 ms)Region opened successfully at 1733732367875 2024-12-09T08:19:27,876 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., pid=9, masterSystemTime=1733732367845 2024-12-09T08:19:27,879 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:27,879 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:27,880 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7a8f9beb16114c9e9ebf1936c0e75d39, regionState=OPEN, openSeqNum=9, regionLocation=5bda946059d1,34083,1733732351576 2024-12-09T08:19:27,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7a8f9beb16114c9e9ebf1936c0e75d39, server=5bda946059d1,34083,1733732351576 because future has completed 2024-12-09T08:19:27,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-09T08:19:27,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 7a8f9beb16114c9e9ebf1936c0e75d39, server=5bda946059d1,34083,1733732351576 in 192 msec 2024-12-09T08:19:27,888 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7a8f9beb16114c9e9ebf1936c0e75d39, REOPEN/MOVE in 584 msec 2024-12-09T08:19:27,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T08:19:27,911 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T08:19:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33313 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:53082 deadline: 1733732427915, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5bda946059d1 port=34083 startCode=1733732351576. As of locationSeqNum=5. 2024-12-09T08:19:27,922 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,33313,1733732351482, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,33313,1733732351482, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5bda946059d1 port=34083 startCode=1733732351576. As of locationSeqNum=5. 2024-12-09T08:19:27,922 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,33313,1733732351482, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5bda946059d1 port=34083 startCode=1733732351576. As of locationSeqNum=5. 2024-12-09T08:19:27,922 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,33313,1733732351482, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,34083,1733732351576, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5bda946059d1 port=34083 startCode=1733732351576. As of locationSeqNum=5. 2024-12-09T08:19:28,035 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T08:19:28,037 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T08:19:28,046 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7a8f9beb16114c9e9ebf1936c0e75d39 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-09T08:19:28,060 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/.tmp/cf1/e6e11479356a4c1388545837da16ba35 is 29, key is r1/cf1:/1733732368038/DeleteFamily/seqid=0 2024-12-09T08:19:28,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741888_1066 (size=4906) 2024-12-09T08:19:28,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741888_1066 (size=4906) 2024-12-09T08:19:28,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741888_1066 (size=4906) 2024-12-09T08:19:28,068 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/.tmp/cf1/e6e11479356a4c1388545837da16ba35 2024-12-09T08:19:28,074 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e6e11479356a4c1388545837da16ba35 2024-12-09T08:19:28,088 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/.tmp/cf2/926de4e6e7904641972b32a17066d1ad is 29, key is r1/cf2:/1733732368038/DeleteFamily/seqid=0 2024-12-09T08:19:28,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741889_1067 (size=4906) 2024-12-09T08:19:28,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741889_1067 (size=4906) 2024-12-09T08:19:28,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741889_1067 (size=4906) 2024-12-09T08:19:28,096 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/.tmp/cf2/926de4e6e7904641972b32a17066d1ad 2024-12-09T08:19:28,101 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 926de4e6e7904641972b32a17066d1ad 2024-12-09T08:19:28,102 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/.tmp/cf1/e6e11479356a4c1388545837da16ba35 as hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf1/e6e11479356a4c1388545837da16ba35 2024-12-09T08:19:28,107 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e6e11479356a4c1388545837da16ba35 2024-12-09T08:19:28,107 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf1/e6e11479356a4c1388545837da16ba35, entries=1, sequenceid=12, filesize=4.8 K 2024-12-09T08:19:28,108 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/.tmp/cf2/926de4e6e7904641972b32a17066d1ad as hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf2/926de4e6e7904641972b32a17066d1ad 2024-12-09T08:19:28,114 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 926de4e6e7904641972b32a17066d1ad 2024-12-09T08:19:28,114 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf2/926de4e6e7904641972b32a17066d1ad, entries=1, sequenceid=12, filesize=4.8 K 2024-12-09T08:19:28,115 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 7a8f9beb16114c9e9ebf1936c0e75d39 in 69ms, sequenceid=12, compaction requested=false 2024-12-09T08:19:28,115 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7a8f9beb16114c9e9ebf1936c0e75d39: 2024-12-09T08:19:28,118 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T08:19:28,119 DEBUG [Time-limited test {}] regionserver.HStore(1541): 7a8f9beb16114c9e9ebf1936c0e75d39/cf1 is initiating major compaction (all files) 2024-12-09T08:19:28,119 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T08:19:28,119 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:28,120 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 7a8f9beb16114c9e9ebf1936c0e75d39/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:28,120 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf1/2129f1563c1e4efe9ea6022bea37442e, hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf1/e6e11479356a4c1388545837da16ba35] into tmpdir=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/.tmp, totalSize=9.5 K 2024-12-09T08:19:28,121 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 2129f1563c1e4efe9ea6022bea37442e, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733732367285 2024-12-09T08:19:28,122 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e6e11479356a4c1388545837da16ba35, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-09T08:19:28,132 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 7a8f9beb16114c9e9ebf1936c0e75d39#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:19:28,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741890_1068 (size=4626) 2024-12-09T08:19:28,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741890_1068 (size=4626) 2024-12-09T08:19:28,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741890_1068 (size=4626) 2024-12-09T08:19:28,145 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/.tmp/cf1/19084ad7b3e94e689b536396901bbd36 as hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf1/19084ad7b3e94e689b536396901bbd36 2024-12-09T08:19:28,158 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 7a8f9beb16114c9e9ebf1936c0e75d39/cf1 of 7a8f9beb16114c9e9ebf1936c0e75d39 into 19084ad7b3e94e689b536396901bbd36(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:19:28,158 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 7a8f9beb16114c9e9ebf1936c0e75d39: 2024-12-09T08:19:28,159 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-09T08:19:28,159 DEBUG [Time-limited test {}] regionserver.HStore(1541): 7a8f9beb16114c9e9ebf1936c0e75d39/cf2 is initiating major compaction (all files) 2024-12-09T08:19:28,159 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T08:19:28,159 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T08:19:28,159 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 7a8f9beb16114c9e9ebf1936c0e75d39/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:28,159 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf2/926de4e6e7904641972b32a17066d1ad] into tmpdir=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/.tmp, totalSize=4.8 K 2024-12-09T08:19:28,160 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 926de4e6e7904641972b32a17066d1ad, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-09T08:19:28,165 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 7a8f9beb16114c9e9ebf1936c0e75d39#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T08:19:28,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741891_1069 (size=4592) 2024-12-09T08:19:28,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741891_1069 (size=4592) 2024-12-09T08:19:28,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741891_1069 (size=4592) 2024-12-09T08:19:28,229 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/.tmp/cf2/a6af77cbba8642c78bea299d6b276ca4 as hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf2/a6af77cbba8642c78bea299d6b276ca4 2024-12-09T08:19:28,236 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 7a8f9beb16114c9e9ebf1936c0e75d39/cf2 of 7a8f9beb16114c9e9ebf1936c0e75d39 into a6af77cbba8642c78bea299d6b276ca4(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:19:28,237 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 7a8f9beb16114c9e9ebf1936c0e75d39: 2024-12-09T08:19:28,241 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=7a8f9beb16114c9e9ebf1936c0e75d39, source=5bda946059d1,34083,1733732351576, destination=5bda946059d1,33313,1733732351482, warming up region on 5bda946059d1,33313,1733732351482 2024-12-09T08:19:28,241 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=7a8f9beb16114c9e9ebf1936c0e75d39, source=5bda946059d1,34083,1733732351576, destination=5bda946059d1,33313,1733732351482, running balancer 2024-12-09T08:19:28,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7a8f9beb16114c9e9ebf1936c0e75d39, REOPEN/MOVE 2024-12-09T08:19:28,243 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7a8f9beb16114c9e9ebf1936c0e75d39, REOPEN/MOVE 2024-12-09T08:19:28,244 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=7a8f9beb16114c9e9ebf1936c0e75d39, regionState=CLOSING, regionLocation=5bda946059d1,34083,1733732351576 2024-12-09T08:19:28,245 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33313 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:28,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33313 {}] regionserver.HRegion(7855): Warmup {ENCODED => 7a8f9beb16114c9e9ebf1936c0e75d39, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:19:28,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33313 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:28,246 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:28,247 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7a8f9beb16114c9e9ebf1936c0e75d39, REOPEN/MOVE because future has completed 2024-12-09T08:19:28,247 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a8f9beb16114c9e9ebf1936c0e75d39 columnFamilyName cf1 2024-12-09T08:19:28,247 DEBUG [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:28,247 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T08:19:28,247 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7a8f9beb16114c9e9ebf1936c0e75d39, server=5bda946059d1,34083,1733732351576}] 2024-12-09T08:19:28,255 DEBUG [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf1/19084ad7b3e94e689b536396901bbd36 2024-12-09T08:19:28,260 DEBUG [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf1/2129f1563c1e4efe9ea6022bea37442e 2024-12-09T08:19:28,264 INFO [StoreFileOpener-7a8f9beb16114c9e9ebf1936c0e75d39-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e6e11479356a4c1388545837da16ba35 2024-12-09T08:19:28,265 DEBUG [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf1/e6e11479356a4c1388545837da16ba35 2024-12-09T08:19:28,265 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.HStore(327): Store=7a8f9beb16114c9e9ebf1936c0e75d39/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:28,265 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:28,266 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a8f9beb16114c9e9ebf1936c0e75d39 columnFamilyName cf2 2024-12-09T08:19:28,266 DEBUG [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:28,273 INFO [StoreFileOpener-7a8f9beb16114c9e9ebf1936c0e75d39-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 926de4e6e7904641972b32a17066d1ad 2024-12-09T08:19:28,273 DEBUG [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf2/926de4e6e7904641972b32a17066d1ad 2024-12-09T08:19:28,278 DEBUG [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf2/a6af77cbba8642c78bea299d6b276ca4 2024-12-09T08:19:28,279 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.HStore(327): Store=7a8f9beb16114c9e9ebf1936c0e75d39/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:28,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33313 {}] regionserver.HRegion(1722): Closing 7a8f9beb16114c9e9ebf1936c0e75d39, disabling compactions & flushes 2024-12-09T08:19:28,279 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33313 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:28,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33313 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:28,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33313 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. after waiting 0 ms 2024-12-09T08:19:28,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33313 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:28,280 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33313 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:28,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33313 {}] regionserver.HRegion(1676): Region close journal for 7a8f9beb16114c9e9ebf1936c0e75d39: Waiting for close lock at 1733732368279Disabling compacts and flushes for region at 1733732368279Disabling writes for close at 1733732368279Writing region close event to WAL at 1733732368280 (+1 ms)Closed at 1733732368280 2024-12-09T08:19:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-09T08:19:28,402 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:28,402 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T08:19:28,402 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 7a8f9beb16114c9e9ebf1936c0e75d39, disabling compactions & flushes 2024-12-09T08:19:28,402 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:28,402 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:28,402 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. after waiting 0 ms 2024-12-09T08:19:28,402 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:28,403 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf1/2129f1563c1e4efe9ea6022bea37442e, hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf1/e6e11479356a4c1388545837da16ba35] to archive 2024-12-09T08:19:28,407 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T08:19:28,411 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf1/2129f1563c1e4efe9ea6022bea37442e to hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf1/2129f1563c1e4efe9ea6022bea37442e 2024-12-09T08:19:28,413 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf1/e6e11479356a4c1388545837da16ba35 to hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf1/e6e11479356a4c1388545837da16ba35 2024-12-09T08:19:28,423 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf2/926de4e6e7904641972b32a17066d1ad] to archive 2024-12-09T08:19:28,424 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T08:19:28,426 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf2/926de4e6e7904641972b32a17066d1ad to hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf2/926de4e6e7904641972b32a17066d1ad 2024-12-09T08:19:28,431 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-09T08:19:28,432 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:28,432 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 7a8f9beb16114c9e9ebf1936c0e75d39: Waiting for close lock at 1733732368402Running coprocessor pre-close hooks at 1733732368402Disabling compacts and flushes for region at 1733732368402Disabling writes for close at 1733732368402Writing region close event to WAL at 1733732368427 (+25 ms)Running coprocessor post-close hooks at 1733732368432 (+5 ms)Closed at 1733732368432 2024-12-09T08:19:28,432 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 7a8f9beb16114c9e9ebf1936c0e75d39 move to 5bda946059d1,33313,1733732351482 record at close sequenceid=12 2024-12-09T08:19:28,434 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:28,435 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=7a8f9beb16114c9e9ebf1936c0e75d39, regionState=CLOSED 2024-12-09T08:19:28,437 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7a8f9beb16114c9e9ebf1936c0e75d39, server=5bda946059d1,34083,1733732351576 because future has completed 2024-12-09T08:19:28,441 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-09T08:19:28,441 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 7a8f9beb16114c9e9ebf1936c0e75d39, server=5bda946059d1,34083,1733732351576 in 192 msec 2024-12-09T08:19:28,442 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7a8f9beb16114c9e9ebf1936c0e75d39, REOPEN/MOVE; state=CLOSED, location=5bda946059d1,33313,1733732351482; forceNewPlan=false, retain=false 2024-12-09T08:19:28,593 INFO [5bda946059d1:41647 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T08:19:28,593 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=7a8f9beb16114c9e9ebf1936c0e75d39, regionState=OPENING, regionLocation=5bda946059d1,33313,1733732351482 2024-12-09T08:19:28,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7a8f9beb16114c9e9ebf1936c0e75d39, REOPEN/MOVE because future has completed 2024-12-09T08:19:28,598 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7a8f9beb16114c9e9ebf1936c0e75d39, server=5bda946059d1,33313,1733732351482}] 2024-12-09T08:19:28,755 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:28,755 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 7a8f9beb16114c9e9ebf1936c0e75d39, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:19:28,756 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:28,756 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:28,756 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:28,756 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:28,757 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:28,758 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a8f9beb16114c9e9ebf1936c0e75d39 columnFamilyName cf1 2024-12-09T08:19:28,758 DEBUG [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:28,765 DEBUG [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf1/19084ad7b3e94e689b536396901bbd36 2024-12-09T08:19:28,765 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.HStore(327): Store=7a8f9beb16114c9e9ebf1936c0e75d39/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:28,765 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:28,766 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a8f9beb16114c9e9ebf1936c0e75d39 columnFamilyName cf2 2024-12-09T08:19:28,766 DEBUG [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:28,773 DEBUG [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf2/a6af77cbba8642c78bea299d6b276ca4 2024-12-09T08:19:28,773 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.HStore(327): Store=7a8f9beb16114c9e9ebf1936c0e75d39/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:28,773 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:28,774 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:28,775 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:28,776 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:28,776 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:28,777 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T08:19:28,778 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:28,778 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 7a8f9beb16114c9e9ebf1936c0e75d39; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72897912, jitterRate=0.08626353740692139}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T08:19:28,779 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:28,779 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 7a8f9beb16114c9e9ebf1936c0e75d39: Running coprocessor pre-open hook at 1733732368756Writing region info on filesystem at 1733732368756Initializing all the Stores at 1733732368757 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732368757Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732368757Cleaning up temporary data from old regions at 1733732368776 (+19 ms)Running coprocessor post-open hooks at 1733732368779 (+3 ms)Region opened successfully at 1733732368779 2024-12-09T08:19:28,780 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., pid=12, masterSystemTime=1733732368751 2024-12-09T08:19:28,783 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:28,783 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:28,784 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=7a8f9beb16114c9e9ebf1936c0e75d39, regionState=OPEN, openSeqNum=18, regionLocation=5bda946059d1,33313,1733732351482 2024-12-09T08:19:28,787 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7a8f9beb16114c9e9ebf1936c0e75d39, server=5bda946059d1,33313,1733732351482 because future has completed 2024-12-09T08:19:28,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-09T08:19:28,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 7a8f9beb16114c9e9ebf1936c0e75d39, server=5bda946059d1,33313,1733732351482 in 190 msec 2024-12-09T08:19:28,793 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7a8f9beb16114c9e9ebf1936c0e75d39, REOPEN/MOVE in 550 msec 2024-12-09T08:19:28,794 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T08:19:28,796 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T08:19:28,798 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 5bda946059d1,33313,1733732351482: testing ***** 2024-12-09T08:19:28,798 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-09T08:19:28,800 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-09T08:19:28,801 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-09T08:19:28,803 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-09T08:19:28,805 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-09T08:19:28,814 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 484569600 }, "NonHeapMemoryUsage": { "committed": 171900928, "init": 7667712, "max": -1, "used": 169197984 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "5bda946059d1", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 0, "ProcessCallTime_max": 9, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 4, "ProcessCallTime_75th_percentile": 6, "ProcessCallTime_90th_percentile": 8, "ProcessCallTime_95th_percentile": 8, "ProcessCallTime_98th_percentile": 8, "ProcessCallTime_99th_percentile": 8, "ProcessCallTime_99.9th_percentile": 8, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 0, "TotalCallTime_max": 9, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 2, "TotalCallTime_median": 4, "TotalCallTime_75th_percentile": 6, "TotalCallTime_90th_percentile": 8, "TotalCallTime_95th_percentile": 8, "TotalCallTime_98th_percentile": 8, "TotalCallTime_99th_percentile": 8, "TotalCallTime_99.9th_percentile": 8, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "5bda946059d1", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:55499", "tag.serverName": "5bda946059d1,33313,1733732351482", "tag.clusterId": "21dc23ac-0dff-402b-b794-53b2dfbe03d4", "tag.Context": "regionserver", "tag.Hostname": "5bda946059d1", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733732351482, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-09T08:19:28,818 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41647 {}] master.MasterRpcServices(700): 5bda946059d1,33313,1733732351482 reported a fatal error: ***** ABORTING region server 5bda946059d1,33313,1733732351482: testing ***** 2024-12-09T08:19:28,821 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5bda946059d1,33313,1733732351482' ***** 2024-12-09T08:19:28,821 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-09T08:19:28,821 INFO [RS:0;5bda946059d1:33313 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T08:19:28,821 INFO [RS:0;5bda946059d1:33313 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-09T08:19:28,821 INFO [RS:0;5bda946059d1:33313 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-09T08:19:28,821 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T08:19:28,821 INFO [RS:0;5bda946059d1:33313 {}] regionserver.HRegionServer(3091): Received CLOSE for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:28,822 INFO [RS:0;5bda946059d1:33313 {}] regionserver.HRegionServer(956): aborting server 5bda946059d1,33313,1733732351482 2024-12-09T08:19:28,822 INFO [RS:0;5bda946059d1:33313 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T08:19:28,822 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7a8f9beb16114c9e9ebf1936c0e75d39, disabling compactions & flushes 2024-12-09T08:19:28,822 INFO [RS:0;5bda946059d1:33313 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;5bda946059d1:33313. 2024-12-09T08:19:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34083 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:56170 deadline: 1733732428822, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5bda946059d1 port=33313 startCode=1733732351482. As of locationSeqNum=12. 2024-12-09T08:19:28,822 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:28,822 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:28,822 DEBUG [RS:0;5bda946059d1:33313 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T08:19:28,822 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. after waiting 0 ms 2024-12-09T08:19:28,822 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:28,823 DEBUG [RS:0;5bda946059d1:33313 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:19:28,823 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,34083,1733732351576, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,34083,1733732351576, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5bda946059d1 port=33313 startCode=1733732351482. As of locationSeqNum=12. 2024-12-09T08:19:28,823 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,34083,1733732351576, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5bda946059d1 port=33313 startCode=1733732351482. As of locationSeqNum=12. 2024-12-09T08:19:28,823 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,34083,1733732351576, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,33313,1733732351482, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=5bda946059d1 port=33313 startCode=1733732351482. As of locationSeqNum=12. 2024-12-09T08:19:28,823 INFO [RS:0;5bda946059d1:33313 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T08:19:28,823 DEBUG [RS:0;5bda946059d1:33313 {}] regionserver.HRegionServer(1325): Online Regions={7a8f9beb16114c9e9ebf1936c0e75d39=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39.} 2024-12-09T08:19:28,823 DEBUG [RS:0;5bda946059d1:33313 {}] regionserver.HRegionServer(1351): Waiting on 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:28,825 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:28,826 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7a8f9beb16114c9e9ebf1936c0e75d39: Waiting for close lock at 1733732368822Running coprocessor pre-close hooks at 1733732368822Disabling compacts and flushes for region at 1733732368822Disabling writes for close at 1733732368822Writing region close event to WAL at 1733732368825 (+3 ms)Running coprocessor post-close hooks at 1733732368825Closed at 1733732368825 2024-12-09T08:19:28,826 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:28,837 INFO [regionserver/5bda946059d1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T08:19:28,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 5bda946059d1,33313,1733732351482 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:28,937 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,33313,1733732351482, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,33313,1733732351482, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 5bda946059d1,33313,1733732351482 aborting 2024-12-09T08:19:28,938 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,33313,1733732351482, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 5bda946059d1,33313,1733732351482 aborting 2024-12-09T08:19:28,938 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,33313,1733732351482, seqNum=12 from cache 2024-12-09T08:19:29,023 INFO [regionserver/5bda946059d1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-09T08:19:29,023 INFO [regionserver/5bda946059d1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-09T08:19:29,024 INFO [RS:0;5bda946059d1:33313 {}] regionserver.HRegionServer(976): stopping server 5bda946059d1,33313,1733732351482; all regions closed. 2024-12-09T08:19:29,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741833_1009 (size=2288) 2024-12-09T08:19:29,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741833_1009 (size=2288) 2024-12-09T08:19:29,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741833_1009 (size=2288) 2024-12-09T08:19:29,030 DEBUG [RS:0;5bda946059d1:33313 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:19:29,030 INFO [RS:0;5bda946059d1:33313 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T08:19:29,030 INFO [RS:0;5bda946059d1:33313 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T08:19:29,030 INFO [RS:0;5bda946059d1:33313 {}] hbase.ChoreService(370): Chore service for: regionserver/5bda946059d1:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T08:19:29,031 INFO [regionserver/5bda946059d1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T08:19:29,031 INFO [RS:0;5bda946059d1:33313 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T08:19:29,031 INFO [RS:0;5bda946059d1:33313 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T08:19:29,031 INFO [RS:0;5bda946059d1:33313 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T08:19:29,031 INFO [RS:0;5bda946059d1:33313 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T08:19:29,032 INFO [RS:0;5bda946059d1:33313 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33313 2024-12-09T08:19:29,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33313-0x10009a83ed90001, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/5bda946059d1,33313,1733732351482 2024-12-09T08:19:29,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T08:19:29,099 INFO [RS:0;5bda946059d1:33313 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T08:19:29,102 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5bda946059d1,33313,1733732351482] 2024-12-09T08:19:29,120 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/5bda946059d1,33313,1733732351482 already deleted, retry=false 2024-12-09T08:19:29,121 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 5bda946059d1,33313,1733732351482 on 5bda946059d1,41647,1733732350732 2024-12-09T08:19:29,128 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 5bda946059d1,33313,1733732351482, splitWal=true, meta=false 2024-12-09T08:19:29,131 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 5bda946059d1,33313,1733732351482 (carryingMeta=false) 5bda946059d1,33313,1733732351482/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@1cbd8d48[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-09T08:19:29,131 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 5bda946059d1,33313,1733732351482, splitWal=true, meta=false 2024-12-09T08:19:29,133 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 5bda946059d1,33313,1733732351482 had 1 regions 2024-12-09T08:19:29,134 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 5bda946059d1,33313,1733732351482, splitWal=true, meta=false, isMeta: false 2024-12-09T08:19:29,136 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,33313,1733732351482-splitting 2024-12-09T08:19:29,137 INFO [PEWorker-2 {}] master.SplitWALManager(105): 5bda946059d1,33313,1733732351482 WAL count=1, meta=false 2024-12-09T08:19:29,139 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 5bda946059d1%2C33313%2C1733732351482.1733732353179}] 2024-12-09T08:19:29,144 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=5bda946059d1,39779,1733732351619 2024-12-09T08:19:29,146 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 5bda946059d1%2C33313%2C1733732351482.1733732353179, worker=5bda946059d1,39779,1733732351619}] 2024-12-09T08:19:29,148 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,33313,1733732351482, seqNum=18] 2024-12-09T08:19:29,150 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 5bda946059d1:33313 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 5bda946059d1/172.17.0.2:33313 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:29,151 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,33313,1733732351482, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,33313,1733732351482, seqNum=18, error=java.net.ConnectException: Call to address=5bda946059d1:33313 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 5bda946059d1/172.17.0.2:33313 2024-12-09T08:19:29,151 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,33313,1733732351482, seqNum=18 is java.net.ConnectException: Connection refused 2024-12-09T08:19:29,151 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,33313,1733732351482, seqNum=18 from cache 2024-12-09T08:19:29,151 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 5bda946059d1:33313 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 5bda946059d1/172.17.0.2:33313 2024-12-09T08:19:29,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33313-0x10009a83ed90001, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T08:19:29,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33313-0x10009a83ed90001, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T08:19:29,211 INFO [RS:0;5bda946059d1:33313 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T08:19:29,211 INFO [RS:0;5bda946059d1:33313 {}] regionserver.HRegionServer(1031): Exiting; stopping=5bda946059d1,33313,1733732351482; zookeeper connection closed. 2024-12-09T08:19:29,211 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@61c45f51 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@61c45f51 2024-12-09T08:19:29,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39779 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-09T08:19:29,323 INFO [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,33313,1733732351482-splitting/5bda946059d1%2C33313%2C1733732351482.1733732353179, size=2.2 K (2288bytes) 2024-12-09T08:19:29,324 INFO [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,33313,1733732351482-splitting/5bda946059d1%2C33313%2C1733732351482.1733732353179 2024-12-09T08:19:29,324 INFO [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,33313,1733732351482-splitting/5bda946059d1%2C33313%2C1733732351482.1733732353179 after 0ms 2024-12-09T08:19:29,348 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:29,348 INFO [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,33313,1733732351482-splitting/5bda946059d1%2C33313%2C1733732351482.1733732353179 took 25ms 2024-12-09T08:19:29,355 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 7a8f9beb16114c9e9ebf1936c0e75d39: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-09T08:19:29,356 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,33313,1733732351482-splitting/5bda946059d1%2C33313%2C1733732351482.1733732353179 so closing down 2024-12-09T08:19:29,356 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:19:29,356 INFO [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:19:29,356 INFO [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 7 ms; skipped=6; WAL=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,33313,1733732351482-splitting/5bda946059d1%2C33313%2C1733732351482.1733732353179, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-12-09T08:19:29,356 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,33313,1733732351482-splitting/5bda946059d1%2C33313%2C1733732351482.1733732353179, journal: Splitting hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,33313,1733732351482-splitting/5bda946059d1%2C33313%2C1733732351482.1733732353179, size=2.2 K (2288bytes) at 1733732369323Finishing writing output for hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,33313,1733732351482-splitting/5bda946059d1%2C33313%2C1733732351482.1733732353179 so closing down at 1733732369356 (+33 ms)3 split writer threads finished at 1733732369356Processed 6 edits across 0 Regions in 7 ms; skipped=6; WAL=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,33313,1733732351482-splitting/5bda946059d1%2C33313%2C1733732351482.1733732353179, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1733732369356 2024-12-09T08:19:29,356 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,33313,1733732351482-splitting/5bda946059d1%2C33313%2C1733732351482.1733732353179 2024-12-09T08:19:29,358 DEBUG [RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-09T08:19:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-09T08:19:29,364 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,33313,1733732351482-splitting/5bda946059d1%2C33313%2C1733732351482.1733732353179 to hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/oldWALs 2024-12-09T08:19:29,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-09T08:19:29,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 5bda946059d1%2C33313%2C1733732351482.1733732353179, worker=5bda946059d1,39779,1733732351619 in 218 msec 2024-12-09T08:19:29,368 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=5bda946059d1,39779,1733732351619 2024-12-09T08:19:29,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-09T08:19:29,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 5bda946059d1%2C33313%2C1733732351482.1733732353179, worker=5bda946059d1,39779,1733732351619 in 231 msec 2024-12-09T08:19:29,372 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,33313,1733732351482-splitting dir is empty, no logs to split. 2024-12-09T08:19:29,372 INFO [PEWorker-1 {}] master.SplitWALManager(105): 5bda946059d1,33313,1733732351482 WAL count=0, meta=false 2024-12-09T08:19:29,373 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 5bda946059d1,33313,1733732351482 WAL splitting is done? wals=0, meta=false 2024-12-09T08:19:29,375 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 5bda946059d1,33313,1733732351482 failed, ignore...File hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/WALs/5bda946059d1,33313,1733732351482-splitting does not exist. 2024-12-09T08:19:29,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7a8f9beb16114c9e9ebf1936c0e75d39, ASSIGN}] 2024-12-09T08:19:29,377 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7a8f9beb16114c9e9ebf1936c0e75d39, ASSIGN 2024-12-09T08:19:29,379 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7a8f9beb16114c9e9ebf1936c0e75d39, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-09T08:19:29,460 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,33313,1733732351482, seqNum=18] 2024-12-09T08:19:29,461 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 5bda946059d1:33313 this server is in the failed servers list 2024-12-09T08:19:29,462 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,33313,1733732351482, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,33313,1733732351482, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=5bda946059d1:33313 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 5bda946059d1:33313 2024-12-09T08:19:29,462 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,33313,1733732351482, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 5bda946059d1:33313 2024-12-09T08:19:29,462 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,33313,1733732351482, seqNum=18 from cache 2024-12-09T08:19:29,529 DEBUG [5bda946059d1:41647 {}] balancer.BalancerClusterState(204): Hosts are {5bda946059d1=0} racks are {/default-rack=0} 2024-12-09T08:19:29,530 DEBUG [5bda946059d1:41647 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T08:19:29,530 DEBUG [5bda946059d1:41647 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T08:19:29,530 DEBUG [5bda946059d1:41647 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T08:19:29,530 DEBUG [5bda946059d1:41647 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T08:19:29,530 INFO [5bda946059d1:41647 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T08:19:29,530 INFO [5bda946059d1:41647 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T08:19:29,530 DEBUG [5bda946059d1:41647 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T08:19:29,531 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=7a8f9beb16114c9e9ebf1936c0e75d39, regionState=OPENING, regionLocation=5bda946059d1,34083,1733732351576 2024-12-09T08:19:29,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7a8f9beb16114c9e9ebf1936c0e75d39, ASSIGN because future has completed 2024-12-09T08:19:29,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7a8f9beb16114c9e9ebf1936c0e75d39, server=5bda946059d1,34083,1733732351576}] 2024-12-09T08:19:29,695 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:29,695 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 7a8f9beb16114c9e9ebf1936c0e75d39, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:19:29,696 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:29,696 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:29,696 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:29,696 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:29,698 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:29,699 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a8f9beb16114c9e9ebf1936c0e75d39 columnFamilyName cf1 2024-12-09T08:19:29,700 DEBUG [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:29,707 DEBUG [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf1/19084ad7b3e94e689b536396901bbd36 2024-12-09T08:19:29,707 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.HStore(327): Store=7a8f9beb16114c9e9ebf1936c0e75d39/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:29,708 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:29,709 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a8f9beb16114c9e9ebf1936c0e75d39 columnFamilyName cf2 2024-12-09T08:19:29,709 DEBUG [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:29,715 DEBUG [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/cf2/a6af77cbba8642c78bea299d6b276ca4 2024-12-09T08:19:29,715 INFO [StoreOpener-7a8f9beb16114c9e9ebf1936c0e75d39-1 {}] regionserver.HStore(327): Store=7a8f9beb16114c9e9ebf1936c0e75d39/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:29,715 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:29,716 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:29,717 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:29,718 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:29,718 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:29,719 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T08:19:29,720 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:29,721 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 7a8f9beb16114c9e9ebf1936c0e75d39; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69357943, jitterRate=0.03351388871669769}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T08:19:29,721 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:29,722 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 7a8f9beb16114c9e9ebf1936c0e75d39: Running coprocessor pre-open hook at 1733732369696Writing region info on filesystem at 1733732369696Initializing all the Stores at 1733732369698 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732369698Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732369698Cleaning up temporary data from old regions at 1733732369718 (+20 ms)Running coprocessor post-open hooks at 1733732369721 (+3 ms)Region opened successfully at 1733732369722 (+1 ms) 2024-12-09T08:19:29,723 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., pid=17, masterSystemTime=1733732369690 2024-12-09T08:19:29,725 DEBUG [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:29,725 INFO [RS_OPEN_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:29,726 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=7a8f9beb16114c9e9ebf1936c0e75d39, regionState=OPEN, openSeqNum=18, regionLocation=5bda946059d1,34083,1733732351576 2024-12-09T08:19:29,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7a8f9beb16114c9e9ebf1936c0e75d39, server=5bda946059d1,34083,1733732351576 because future has completed 2024-12-09T08:19:29,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-09T08:19:29,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 7a8f9beb16114c9e9ebf1936c0e75d39, server=5bda946059d1,34083,1733732351576 in 193 msec 2024-12-09T08:19:29,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-09T08:19:29,735 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 5bda946059d1,33313,1733732351482 after splitting done 2024-12-09T08:19:29,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7a8f9beb16114c9e9ebf1936c0e75d39, ASSIGN in 356 msec 2024-12-09T08:19:29,737 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 5bda946059d1,33313,1733732351482, splitWal=true, meta=false in 613 msec 2024-12-09T08:19:29,969 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39., hostname=5bda946059d1,34083,1733732351576, seqNum=18] 2024-12-09T08:19:29,986 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=408 (was 407) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1683044924_22 at /127.0.0.1:57414 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/cluster_95cc11f3-affe-6e07-bfcd-38f5721bf4f5/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-604705958_22 at /127.0.0.1:40508 [Waiting for operation #28] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_LOG_REPLAY_OPS-regionserver/5bda946059d1:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1683044924_22 at /127.0.0.1:55428 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/5bda946059d1:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/cluster_95cc11f3-affe-6e07-bfcd-38f5721bf4f5/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1051 (was 1022) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=234 (was 237), ProcessCount=11 (was 11), AvailableMemoryMB=4350 (was 4405) 2024-12-09T08:19:29,988 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-12-09T08:19:30,000 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=408, OpenFileDescriptor=1051, MaxFileDescriptor=1048576, SystemLoadAverage=234, ProcessCount=11, AvailableMemoryMB=4349 2024-12-09T08:19:30,000 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-12-09T08:19:30,015 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:30,017 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:30,018 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:19:30,020 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-58173813, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/hregion-58173813, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:30,031 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-58173813/hregion-58173813.1733732370020, exclude list is [], retry=0 2024-12-09T08:19:30,034 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:30,034 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:30,034 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:30,057 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-58173813/hregion-58173813.1733732370020 2024-12-09T08:19:30,058 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41283:41283),(127.0.0.1/127.0.0.1:42925:42925),(127.0.0.1/127.0.0.1:42683:42683)] 2024-12-09T08:19:30,058 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 850922825e0f224d500bf5ccf2b323ef, NAME => 'testReplayEditsWrittenViaHRegion,,1733732370016.850922825e0f224d500bf5ccf2b323ef.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45647/hbase 2024-12-09T08:19:30,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741893_1071 (size=67) 2024-12-09T08:19:30,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741893_1071 (size=67) 2024-12-09T08:19:30,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741893_1071 (size=67) 2024-12-09T08:19:30,069 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733732370016.850922825e0f224d500bf5ccf2b323ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:30,070 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,071 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 850922825e0f224d500bf5ccf2b323ef columnFamilyName a 2024-12-09T08:19:30,072 DEBUG [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:30,072 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] regionserver.HStore(327): Store=850922825e0f224d500bf5ccf2b323ef/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:30,072 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,073 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 850922825e0f224d500bf5ccf2b323ef columnFamilyName b 2024-12-09T08:19:30,073 DEBUG [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:30,074 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] regionserver.HStore(327): Store=850922825e0f224d500bf5ccf2b323ef/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:30,074 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,075 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 850922825e0f224d500bf5ccf2b323ef columnFamilyName c 2024-12-09T08:19:30,075 DEBUG [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:30,076 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] regionserver.HStore(327): Store=850922825e0f224d500bf5ccf2b323ef/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:30,076 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,077 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,077 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,078 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,078 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,079 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:19:30,080 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,082 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:19:30,082 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 850922825e0f224d500bf5ccf2b323ef; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59077042, jitterRate=-0.11968347430229187}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:19:30,083 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 850922825e0f224d500bf5ccf2b323ef: Writing region info on filesystem at 1733732370069Initializing all the Stores at 1733732370070 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732370070Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732370070Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732370070Cleaning up temporary data from old regions at 1733732370078 (+8 ms)Region opened successfully at 1733732370083 (+5 ms) 2024-12-09T08:19:30,083 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 850922825e0f224d500bf5ccf2b323ef, disabling compactions & flushes 2024-12-09T08:19:30,083 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733732370016.850922825e0f224d500bf5ccf2b323ef. 2024-12-09T08:19:30,083 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733732370016.850922825e0f224d500bf5ccf2b323ef. 2024-12-09T08:19:30,083 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733732370016.850922825e0f224d500bf5ccf2b323ef. after waiting 0 ms 2024-12-09T08:19:30,083 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733732370016.850922825e0f224d500bf5ccf2b323ef. 2024-12-09T08:19:30,084 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733732370016.850922825e0f224d500bf5ccf2b323ef. 2024-12-09T08:19:30,084 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 850922825e0f224d500bf5ccf2b323ef: Waiting for close lock at 1733732370083Disabling compacts and flushes for region at 1733732370083Disabling writes for close at 1733732370083Writing region close event to WAL at 1733732370084 (+1 ms)Closed at 1733732370084 2024-12-09T08:19:30,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741892_1070 (size=175) 2024-12-09T08:19:30,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741892_1070 (size=175) 2024-12-09T08:19:30,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741892_1070 (size=175) 2024-12-09T08:19:30,089 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:19:30,089 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-58173813:(num 1733732370020) 2024-12-09T08:19:30,089 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:19:30,091 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732370015, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:30,103 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732370015/wal.1733732370092, exclude list is [], retry=0 2024-12-09T08:19:30,106 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:30,106 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:30,106 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:30,130 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732370015/wal.1733732370092 2024-12-09T08:19:30,130 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42683:42683),(127.0.0.1/127.0.0.1:41283:41283),(127.0.0.1/127.0.0.1:42925:42925)] 2024-12-09T08:19:30,130 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 850922825e0f224d500bf5ccf2b323ef, NAME => 'testReplayEditsWrittenViaHRegion,,1733732370016.850922825e0f224d500bf5ccf2b323ef.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:19:30,130 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733732370016.850922825e0f224d500bf5ccf2b323ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:30,130 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,130 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,132 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,133 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 850922825e0f224d500bf5ccf2b323ef columnFamilyName a 2024-12-09T08:19:30,133 DEBUG [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:30,133 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] regionserver.HStore(327): Store=850922825e0f224d500bf5ccf2b323ef/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:30,134 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,134 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 850922825e0f224d500bf5ccf2b323ef columnFamilyName b 2024-12-09T08:19:30,134 DEBUG [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:30,135 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] regionserver.HStore(327): Store=850922825e0f224d500bf5ccf2b323ef/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:30,135 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,136 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 850922825e0f224d500bf5ccf2b323ef columnFamilyName c 2024-12-09T08:19:30,136 DEBUG [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:30,136 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] regionserver.HStore(327): Store=850922825e0f224d500bf5ccf2b323ef/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:30,136 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,137 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,138 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,139 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,139 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,140 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:19:30,141 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,142 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 850922825e0f224d500bf5ccf2b323ef; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67186472, jitterRate=0.001156449317932129}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:19:30,143 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 850922825e0f224d500bf5ccf2b323ef: Writing region info on filesystem at 1733732370131Initializing all the Stores at 1733732370132 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732370132Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732370132Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732370132Cleaning up temporary data from old regions at 1733732370139 (+7 ms)Region opened successfully at 1733732370143 (+4 ms) 2024-12-09T08:19:30,166 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 850922825e0f224d500bf5ccf2b323ef 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-09T08:19:30,181 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/.tmp/a/613e337b053e44cc88d8f01632f560d7 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733732370143/Put/seqid=0 2024-12-09T08:19:30,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741895_1073 (size=5958) 2024-12-09T08:19:30,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741895_1073 (size=5958) 2024-12-09T08:19:30,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741895_1073 (size=5958) 2024-12-09T08:19:30,189 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/.tmp/a/613e337b053e44cc88d8f01632f560d7 2024-12-09T08:19:30,207 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/.tmp/b/ea4aba2164ae47edad69ad795c32ebac is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733732370150/Put/seqid=0 2024-12-09T08:19:30,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741896_1074 (size=5958) 2024-12-09T08:19:30,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741896_1074 (size=5958) 2024-12-09T08:19:30,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741896_1074 (size=5958) 2024-12-09T08:19:30,214 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/.tmp/b/ea4aba2164ae47edad69ad795c32ebac 2024-12-09T08:19:30,236 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/.tmp/c/b4970beddf5c4bdc9975fc52023e1d8c is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733732370157/Put/seqid=0 2024-12-09T08:19:30,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741897_1075 (size=5958) 2024-12-09T08:19:30,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741897_1075 (size=5958) 2024-12-09T08:19:30,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741897_1075 (size=5958) 2024-12-09T08:19:30,245 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/.tmp/c/b4970beddf5c4bdc9975fc52023e1d8c 2024-12-09T08:19:30,253 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/.tmp/a/613e337b053e44cc88d8f01632f560d7 as hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/a/613e337b053e44cc88d8f01632f560d7 2024-12-09T08:19:30,260 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/a/613e337b053e44cc88d8f01632f560d7, entries=10, sequenceid=33, filesize=5.8 K 2024-12-09T08:19:30,261 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/.tmp/b/ea4aba2164ae47edad69ad795c32ebac as hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/b/ea4aba2164ae47edad69ad795c32ebac 2024-12-09T08:19:30,268 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/b/ea4aba2164ae47edad69ad795c32ebac, entries=10, sequenceid=33, filesize=5.8 K 2024-12-09T08:19:30,270 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/.tmp/c/b4970beddf5c4bdc9975fc52023e1d8c as hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/c/b4970beddf5c4bdc9975fc52023e1d8c 2024-12-09T08:19:30,276 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/c/b4970beddf5c4bdc9975fc52023e1d8c, entries=10, sequenceid=33, filesize=5.8 K 2024-12-09T08:19:30,277 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 850922825e0f224d500bf5ccf2b323ef in 111ms, sequenceid=33, compaction requested=false 2024-12-09T08:19:30,277 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 850922825e0f224d500bf5ccf2b323ef: 2024-12-09T08:19:30,277 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 850922825e0f224d500bf5ccf2b323ef, disabling compactions & flushes 2024-12-09T08:19:30,277 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733732370016.850922825e0f224d500bf5ccf2b323ef. 2024-12-09T08:19:30,277 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733732370016.850922825e0f224d500bf5ccf2b323ef. 2024-12-09T08:19:30,278 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733732370016.850922825e0f224d500bf5ccf2b323ef. after waiting 0 ms 2024-12-09T08:19:30,278 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733732370016.850922825e0f224d500bf5ccf2b323ef. 2024-12-09T08:19:30,279 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733732370016.850922825e0f224d500bf5ccf2b323ef. 2024-12-09T08:19:30,279 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 850922825e0f224d500bf5ccf2b323ef: Waiting for close lock at 1733732370277Disabling compacts and flushes for region at 1733732370277Disabling writes for close at 1733732370278 (+1 ms)Writing region close event to WAL at 1733732370279 (+1 ms)Closed at 1733732370279 2024-12-09T08:19:30,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741894_1072 (size=6311) 2024-12-09T08:19:30,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741894_1072 (size=6311) 2024-12-09T08:19:30,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741894_1072 (size=6311) 2024-12-09T08:19:30,287 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/b/ea4aba2164ae47edad69ad795c32ebac to hdfs://localhost:45647/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/b/ea4aba2164ae47edad69ad795c32ebac 2024-12-09T08:19:30,310 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732370015/wal.1733732370092, size=6.2 K (6311bytes) 2024-12-09T08:19:30,310 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732370015/wal.1733732370092 2024-12-09T08:19:30,310 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732370015/wal.1733732370092 after 0ms 2024-12-09T08:19:30,335 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:30,335 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732370015/wal.1733732370092 took 25ms 2024-12-09T08:19:30,337 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732370015/wal.1733732370092 so closing down 2024-12-09T08:19:30,337 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:19:30,338 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733732370092.temp 2024-12-09T08:19:30,362 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/recovered.edits/0000000000000000003-wal.1733732370092.temp 2024-12-09T08:19:30,363 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:19:30,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741898_1076 (size=5575) 2024-12-09T08:19:30,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741898_1076 (size=5575) 2024-12-09T08:19:30,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741898_1076 (size=5575) 2024-12-09T08:19:30,369 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/recovered.edits/0000000000000000003-wal.1733732370092.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-09T08:19:30,370 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/recovered.edits/0000000000000000003-wal.1733732370092.temp to hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/recovered.edits/0000000000000000032 2024-12-09T08:19:30,370 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 35 ms; skipped=2; WAL=hdfs://localhost:45647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732370015/wal.1733732370092, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-12-09T08:19:30,370 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732370015/wal.1733732370092, journal: Splitting hdfs://localhost:45647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732370015/wal.1733732370092, size=6.2 K (6311bytes) at 1733732370310Finishing writing output for hdfs://localhost:45647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732370015/wal.1733732370092 so closing down at 1733732370337 (+27 ms)Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/recovered.edits/0000000000000000003-wal.1733732370092.temp at 1733732370362 (+25 ms)3 split writer threads finished at 1733732370363 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/recovered.edits/0000000000000000003-wal.1733732370092.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733732370369 (+6 ms)Rename recovered edits hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/recovered.edits/0000000000000000003-wal.1733732370092.temp to hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/recovered.edits/0000000000000000032 at 1733732370370 (+1 ms)Processed 32 edits across 1 Regions in 35 ms; skipped=2; WAL=hdfs://localhost:45647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732370015/wal.1733732370092, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1733732370370 2024-12-09T08:19:30,372 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732370015/wal.1733732370092 to hdfs://localhost:45647/hbase/oldWALs/wal.1733732370092 2024-12-09T08:19:30,372 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/recovered.edits/0000000000000000032 2024-12-09T08:19:30,372 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:19:30,374 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732370015, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:30,384 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732370015/wal.1733732370374, exclude list is [], retry=0 2024-12-09T08:19:30,386 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:30,387 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:30,387 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:30,407 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732370015/wal.1733732370374 2024-12-09T08:19:30,407 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42925:42925),(127.0.0.1/127.0.0.1:42683:42683),(127.0.0.1/127.0.0.1:41283:41283)] 2024-12-09T08:19:30,407 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 850922825e0f224d500bf5ccf2b323ef, NAME => 'testReplayEditsWrittenViaHRegion,,1733732370016.850922825e0f224d500bf5ccf2b323ef.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:19:30,408 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733732370016.850922825e0f224d500bf5ccf2b323ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:30,408 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,408 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,409 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,410 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 850922825e0f224d500bf5ccf2b323ef columnFamilyName a 2024-12-09T08:19:30,410 DEBUG [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:30,415 DEBUG [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/a/613e337b053e44cc88d8f01632f560d7 2024-12-09T08:19:30,415 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] regionserver.HStore(327): Store=850922825e0f224d500bf5ccf2b323ef/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:30,415 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,416 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 850922825e0f224d500bf5ccf2b323ef columnFamilyName b 2024-12-09T08:19:30,416 DEBUG [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:30,417 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] regionserver.HStore(327): Store=850922825e0f224d500bf5ccf2b323ef/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:30,417 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,417 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 850922825e0f224d500bf5ccf2b323ef columnFamilyName c 2024-12-09T08:19:30,417 DEBUG [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:30,423 DEBUG [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/c/b4970beddf5c4bdc9975fc52023e1d8c 2024-12-09T08:19:30,423 INFO [StoreOpener-850922825e0f224d500bf5ccf2b323ef-1 {}] regionserver.HStore(327): Store=850922825e0f224d500bf5ccf2b323ef/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:30,424 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,424 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,426 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,426 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/recovered.edits/0000000000000000032 2024-12-09T08:19:30,453 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:30,455 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/recovered.edits/0000000000000000032 2024-12-09T08:19:30,456 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 850922825e0f224d500bf5ccf2b323ef 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-09T08:19:30,469 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/.tmp/b/a322fbc95f4f4ce6859f45067aeb97b7 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733732370150/Put/seqid=0 2024-12-09T08:19:30,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741900_1078 (size=5958) 2024-12-09T08:19:30,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741900_1078 (size=5958) 2024-12-09T08:19:30,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741900_1078 (size=5958) 2024-12-09T08:19:30,477 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/.tmp/b/a322fbc95f4f4ce6859f45067aeb97b7 2024-12-09T08:19:30,485 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/.tmp/b/a322fbc95f4f4ce6859f45067aeb97b7 as hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/b/a322fbc95f4f4ce6859f45067aeb97b7 2024-12-09T08:19:30,490 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/b/a322fbc95f4f4ce6859f45067aeb97b7, entries=10, sequenceid=32, filesize=5.8 K 2024-12-09T08:19:30,490 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 850922825e0f224d500bf5ccf2b323ef in 35ms, sequenceid=32, compaction requested=false; wal=null 2024-12-09T08:19:30,491 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/recovered.edits/0000000000000000032 2024-12-09T08:19:30,492 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,492 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,493 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:19:30,494 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 850922825e0f224d500bf5ccf2b323ef 2024-12-09T08:19:30,496 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/850922825e0f224d500bf5ccf2b323ef/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-09T08:19:30,497 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 850922825e0f224d500bf5ccf2b323ef; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73772305, jitterRate=0.09929300844669342}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:19:30,498 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 850922825e0f224d500bf5ccf2b323ef: Writing region info on filesystem at 1733732370408Initializing all the Stores at 1733732370409 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732370409Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732370409Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732370409Obtaining lock to block concurrent updates at 1733732370456 (+47 ms)Preparing flush snapshotting stores in 850922825e0f224d500bf5ccf2b323ef at 1733732370456Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733732370016.850922825e0f224d500bf5ccf2b323ef., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733732370456Flushing stores of testReplayEditsWrittenViaHRegion,,1733732370016.850922825e0f224d500bf5ccf2b323ef. at 1733732370456Flushing 850922825e0f224d500bf5ccf2b323ef/b: creating writer at 1733732370456Flushing 850922825e0f224d500bf5ccf2b323ef/b: appending metadata at 1733732370468 (+12 ms)Flushing 850922825e0f224d500bf5ccf2b323ef/b: closing flushed file at 1733732370468Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15483444: reopening flushed file at 1733732370484 (+16 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 850922825e0f224d500bf5ccf2b323ef in 35ms, sequenceid=32, compaction requested=false; wal=null at 1733732370490 (+6 ms)Cleaning up temporary data from old regions at 1733732370492 (+2 ms)Region opened successfully at 1733732370498 (+6 ms) 2024-12-09T08:19:30,520 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=417 (was 408) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:57414 [Waiting for operation #32] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1712704723-172.17.0.2-1733732346335:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:36134 [Receiving block BP-1712704723-172.17.0.2-1733732346335:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:40508 [Waiting for operation #32] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:35002 [Receiving block BP-1712704723-172.17.0.2-1733732346335:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:55428 [Waiting for operation #38] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1712704723-172.17.0.2-1733732346335:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1712704723-172.17.0.2-1733732346335:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:58436 [Receiving block BP-1712704723-172.17.0.2-1733732346335:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1127 (was 1051) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=234 (was 234), ProcessCount=11 (was 11), AvailableMemoryMB=4327 (was 4349) 2024-12-09T08:19:30,520 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1127 is superior to 1024 2024-12-09T08:19:30,532 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=417, OpenFileDescriptor=1127, MaxFileDescriptor=1048576, SystemLoadAverage=234, ProcessCount=11, AvailableMemoryMB=4326 2024-12-09T08:19:30,532 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1127 is superior to 1024 2024-12-09T08:19:30,551 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:30,553 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:30,554 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:19:30,557 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-98477708, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/hregion-98477708, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:30,573 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-98477708/hregion-98477708.1733732370557, exclude list is [], retry=0 2024-12-09T08:19:30,576 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:30,576 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:30,576 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:30,600 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-98477708/hregion-98477708.1733732370557 2024-12-09T08:19:30,601 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42925:42925),(127.0.0.1/127.0.0.1:41283:41283),(127.0.0.1/127.0.0.1:42683:42683)] 2024-12-09T08:19:30,601 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a4d0037a191e85107548afaafaf2f7b3, NAME => 'testReplayEditsAfterAbortingFlush,,1733732370552.a4d0037a191e85107548afaafaf2f7b3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45647/hbase 2024-12-09T08:19:30,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741902_1080 (size=68) 2024-12-09T08:19:30,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741902_1080 (size=68) 2024-12-09T08:19:30,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741902_1080 (size=68) 2024-12-09T08:19:30,609 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733732370552.a4d0037a191e85107548afaafaf2f7b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:30,610 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:30,611 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4d0037a191e85107548afaafaf2f7b3 columnFamilyName a 2024-12-09T08:19:30,611 DEBUG [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:30,612 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] regionserver.HStore(327): Store=a4d0037a191e85107548afaafaf2f7b3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:30,612 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:30,613 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4d0037a191e85107548afaafaf2f7b3 columnFamilyName b 2024-12-09T08:19:30,613 DEBUG [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:30,613 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] regionserver.HStore(327): Store=a4d0037a191e85107548afaafaf2f7b3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:30,613 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:30,614 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4d0037a191e85107548afaafaf2f7b3 columnFamilyName c 2024-12-09T08:19:30,615 DEBUG [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:30,615 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] regionserver.HStore(327): Store=a4d0037a191e85107548afaafaf2f7b3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:30,615 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:30,616 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:30,616 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:30,617 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:30,617 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:30,617 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:19:30,618 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:30,620 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:19:30,621 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a4d0037a191e85107548afaafaf2f7b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62691164, jitterRate=-0.06582885980606079}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:19:30,621 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a4d0037a191e85107548afaafaf2f7b3: Writing region info on filesystem at 1733732370609Initializing all the Stores at 1733732370610 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732370610Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732370610Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732370610Cleaning up temporary data from old regions at 1733732370617 (+7 ms)Region opened successfully at 1733732370621 (+4 ms) 2024-12-09T08:19:30,621 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a4d0037a191e85107548afaafaf2f7b3, disabling compactions & flushes 2024-12-09T08:19:30,621 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733732370552.a4d0037a191e85107548afaafaf2f7b3. 2024-12-09T08:19:30,621 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733732370552.a4d0037a191e85107548afaafaf2f7b3. 2024-12-09T08:19:30,621 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733732370552.a4d0037a191e85107548afaafaf2f7b3. after waiting 0 ms 2024-12-09T08:19:30,622 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733732370552.a4d0037a191e85107548afaafaf2f7b3. 2024-12-09T08:19:30,622 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733732370552.a4d0037a191e85107548afaafaf2f7b3. 2024-12-09T08:19:30,622 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a4d0037a191e85107548afaafaf2f7b3: Waiting for close lock at 1733732370621Disabling compacts and flushes for region at 1733732370621Disabling writes for close at 1733732370621Writing region close event to WAL at 1733732370622 (+1 ms)Closed at 1733732370622 2024-12-09T08:19:30,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741901_1079 (size=175) 2024-12-09T08:19:30,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741901_1079 (size=175) 2024-12-09T08:19:30,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741901_1079 (size=175) 2024-12-09T08:19:30,626 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:19:30,626 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-98477708:(num 1733732370557) 2024-12-09T08:19:30,627 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:19:30,628 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732370550, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:30,639 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732370550/wal.1733732370628, exclude list is [], retry=0 2024-12-09T08:19:30,641 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:30,642 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:30,642 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:30,673 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732370550/wal.1733732370628 2024-12-09T08:19:30,673 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42683:42683),(127.0.0.1/127.0.0.1:42925:42925),(127.0.0.1/127.0.0.1:41283:41283)] 2024-12-09T08:19:30,733 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a4d0037a191e85107548afaafaf2f7b3, NAME => 'testReplayEditsAfterAbortingFlush,,1733732370552.a4d0037a191e85107548afaafaf2f7b3.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:19:30,735 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:30,735 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733732370552.a4d0037a191e85107548afaafaf2f7b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:30,735 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:30,735 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:30,737 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:30,738 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4d0037a191e85107548afaafaf2f7b3 columnFamilyName a 2024-12-09T08:19:30,738 DEBUG [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:30,739 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] regionserver.HStore(327): Store=a4d0037a191e85107548afaafaf2f7b3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:30,739 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:30,740 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4d0037a191e85107548afaafaf2f7b3 columnFamilyName b 2024-12-09T08:19:30,740 DEBUG [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:30,740 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] regionserver.HStore(327): Store=a4d0037a191e85107548afaafaf2f7b3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:30,740 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:30,741 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4d0037a191e85107548afaafaf2f7b3 columnFamilyName c 2024-12-09T08:19:30,741 DEBUG [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:30,741 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] regionserver.HStore(327): Store=a4d0037a191e85107548afaafaf2f7b3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:30,742 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:30,742 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:30,744 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:30,745 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:30,745 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:30,745 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:19:30,747 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:30,748 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a4d0037a191e85107548afaafaf2f7b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64397118, jitterRate=-0.040408164262771606}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:19:30,748 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:30,748 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a4d0037a191e85107548afaafaf2f7b3: Running coprocessor pre-open hook at 1733732370735Writing region info on filesystem at 1733732370735Initializing all the Stores at 1733732370737 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732370737Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732370737Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732370737Cleaning up temporary data from old regions at 1733732370745 (+8 ms)Running coprocessor post-open hooks at 1733732370748 (+3 ms)Region opened successfully at 1733732370748 2024-12-09T08:19:30,761 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a4d0037a191e85107548afaafaf2f7b3 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-09T08:19:30,762 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a4d0037a191e85107548afaafaf2f7b3/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:31,066 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-09T08:19:31,067 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-09T08:19:31,067 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-09T08:19:31,068 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-09T08:19:31,068 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-09T08:19:31,068 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-09T08:19:31,069 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-09T08:19:31,069 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-09T08:19:31,070 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-09T08:19:31,070 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-09T08:19:31,763 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a4d0037a191e85107548afaafaf2f7b3/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:32,526 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T08:19:32,764 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a4d0037a191e85107548afaafaf2f7b3/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:33,764 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a4d0037a191e85107548afaafaf2f7b3/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:34,765 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a4d0037a191e85107548afaafaf2f7b3/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:35,766 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a4d0037a191e85107548afaafaf2f7b3/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:36,767 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a4d0037a191e85107548afaafaf2f7b3/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:37,770 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a4d0037a191e85107548afaafaf2f7b3/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:38,771 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a4d0037a191e85107548afaafaf2f7b3/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:39,772 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for a4d0037a191e85107548afaafaf2f7b3/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:39,775 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a4d0037a191e85107548afaafaf2f7b3: 2024-12-09T08:19:39,775 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733732370552.a4d0037a191e85107548afaafaf2f7b3. 2024-12-09T08:19:39,794 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a4d0037a191e85107548afaafaf2f7b3: 2024-12-09T08:19:39,794 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-09T08:19:39,794 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a4d0037a191e85107548afaafaf2f7b3, disabling compactions & flushes 2024-12-09T08:19:39,794 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733732370552.a4d0037a191e85107548afaafaf2f7b3. 2024-12-09T08:19:39,794 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733732370552.a4d0037a191e85107548afaafaf2f7b3. 2024-12-09T08:19:39,794 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733732370552.a4d0037a191e85107548afaafaf2f7b3. after waiting 0 ms 2024-12-09T08:19:39,794 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733732370552.a4d0037a191e85107548afaafaf2f7b3. 2024-12-09T08:19:39,794 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733732370552.a4d0037a191e85107548afaafaf2f7b3. 2024-12-09T08:19:39,795 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733732370552.a4d0037a191e85107548afaafaf2f7b3. 2024-12-09T08:19:39,795 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a4d0037a191e85107548afaafaf2f7b3: Waiting for close lock at 1733732379794Running coprocessor pre-close hooks at 1733732379794Disabling compacts and flushes for region at 1733732379794Disabling writes for close at 1733732379794Writing region close event to WAL at 1733732379794Running coprocessor post-close hooks at 1733732379794Closed at 1733732379794 2024-12-09T08:19:39,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741903_1081 (size=4236) 2024-12-09T08:19:39,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741903_1081 (size=4236) 2024-12-09T08:19:39,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741903_1081 (size=4236) 2024-12-09T08:19:39,811 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732370550/wal.1733732370628, size=4.1 K (4236bytes) 2024-12-09T08:19:39,811 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732370550/wal.1733732370628 2024-12-09T08:19:39,811 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732370550/wal.1733732370628 after 0ms 2024-12-09T08:19:39,836 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:39,837 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732370550/wal.1733732370628 took 27ms 2024-12-09T08:19:39,839 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732370550/wal.1733732370628 so closing down 2024-12-09T08:19:39,839 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:19:39,840 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733732370628.temp 2024-12-09T08:19:39,862 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/recovered.edits/0000000000000000004-wal.1733732370628.temp 2024-12-09T08:19:39,862 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:19:39,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741904_1082 (size=3245) 2024-12-09T08:19:39,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741904_1082 (size=3245) 2024-12-09T08:19:39,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741904_1082 (size=3245) 2024-12-09T08:19:39,870 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/recovered.edits/0000000000000000004-wal.1733732370628.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-09T08:19:39,871 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/recovered.edits/0000000000000000004-wal.1733732370628.temp to hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/recovered.edits/0000000000000000026 2024-12-09T08:19:39,871 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 34 ms; skipped=3; WAL=hdfs://localhost:45647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732370550/wal.1733732370628, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-12-09T08:19:39,871 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732370550/wal.1733732370628, journal: Splitting hdfs://localhost:45647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732370550/wal.1733732370628, size=4.1 K (4236bytes) at 1733732379811Finishing writing output for hdfs://localhost:45647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732370550/wal.1733732370628 so closing down at 1733732379839 (+28 ms)Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/recovered.edits/0000000000000000004-wal.1733732370628.temp at 1733732379862 (+23 ms)3 split writer threads finished at 1733732379862Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/recovered.edits/0000000000000000004-wal.1733732370628.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733732379870 (+8 ms)Rename recovered edits hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/recovered.edits/0000000000000000004-wal.1733732370628.temp to hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/recovered.edits/0000000000000000026 at 1733732379871 (+1 ms)Processed 23 edits across 1 Regions in 34 ms; skipped=3; WAL=hdfs://localhost:45647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732370550/wal.1733732370628, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1733732379871 2024-12-09T08:19:39,872 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732370550/wal.1733732370628 to hdfs://localhost:45647/hbase/oldWALs/wal.1733732370628 2024-12-09T08:19:39,873 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/recovered.edits/0000000000000000026 2024-12-09T08:19:39,873 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:19:39,875 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732370550, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:39,886 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732370550/wal.1733732379875, exclude list is [], retry=0 2024-12-09T08:19:39,888 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:39,889 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:39,889 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:39,911 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732370550/wal.1733732379875 2024-12-09T08:19:39,911 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41283:41283),(127.0.0.1/127.0.0.1:42925:42925),(127.0.0.1/127.0.0.1:42683:42683)] 2024-12-09T08:19:39,912 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a4d0037a191e85107548afaafaf2f7b3, NAME => 'testReplayEditsAfterAbortingFlush,,1733732370552.a4d0037a191e85107548afaafaf2f7b3.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:19:39,912 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:39,912 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733732370552.a4d0037a191e85107548afaafaf2f7b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:39,912 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:39,912 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:39,914 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:39,915 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4d0037a191e85107548afaafaf2f7b3 columnFamilyName a 2024-12-09T08:19:39,915 DEBUG [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:39,916 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] regionserver.HStore(327): Store=a4d0037a191e85107548afaafaf2f7b3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:39,916 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:39,916 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4d0037a191e85107548afaafaf2f7b3 columnFamilyName b 2024-12-09T08:19:39,916 DEBUG [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:39,917 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] regionserver.HStore(327): Store=a4d0037a191e85107548afaafaf2f7b3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:39,917 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:39,918 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4d0037a191e85107548afaafaf2f7b3 columnFamilyName c 2024-12-09T08:19:39,918 DEBUG [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:39,918 INFO [StoreOpener-a4d0037a191e85107548afaafaf2f7b3-1 {}] regionserver.HStore(327): Store=a4d0037a191e85107548afaafaf2f7b3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:39,918 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:39,919 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:39,921 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:39,921 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/recovered.edits/0000000000000000026 2024-12-09T08:19:39,928 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T08:19:39,945 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:39,946 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/recovered.edits/0000000000000000026 2024-12-09T08:19:39,947 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a4d0037a191e85107548afaafaf2f7b3 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-09T08:19:39,960 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/.tmp/a/2456c0a7e7b2473e88141fd6c894044b is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733732379783/Put/seqid=0 2024-12-09T08:19:39,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741906_1084 (size=5523) 2024-12-09T08:19:39,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741906_1084 (size=5523) 2024-12-09T08:19:39,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741906_1084 (size=5523) 2024-12-09T08:19:39,966 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/.tmp/a/2456c0a7e7b2473e88141fd6c894044b 2024-12-09T08:19:39,984 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/.tmp/b/4fee65392ffc441e9404d044591c4cca is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733732379776/Put/seqid=0 2024-12-09T08:19:39,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741907_1085 (size=5524) 2024-12-09T08:19:39,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741907_1085 (size=5524) 2024-12-09T08:19:39,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741907_1085 (size=5524) 2024-12-09T08:19:39,990 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/.tmp/b/4fee65392ffc441e9404d044591c4cca 2024-12-09T08:19:40,009 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/.tmp/c/81db6571d5254ce284092c4dda1e151f is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733732379781/Put/seqid=0 2024-12-09T08:19:40,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741908_1086 (size=5457) 2024-12-09T08:19:40,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741908_1086 (size=5457) 2024-12-09T08:19:40,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741908_1086 (size=5457) 2024-12-09T08:19:40,020 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/.tmp/c/81db6571d5254ce284092c4dda1e151f 2024-12-09T08:19:40,026 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/.tmp/a/2456c0a7e7b2473e88141fd6c894044b as hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/a/2456c0a7e7b2473e88141fd6c894044b 2024-12-09T08:19:40,032 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/a/2456c0a7e7b2473e88141fd6c894044b, entries=7, sequenceid=26, filesize=5.4 K 2024-12-09T08:19:40,033 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/.tmp/b/4fee65392ffc441e9404d044591c4cca as hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/b/4fee65392ffc441e9404d044591c4cca 2024-12-09T08:19:40,039 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/b/4fee65392ffc441e9404d044591c4cca, entries=7, sequenceid=26, filesize=5.4 K 2024-12-09T08:19:40,040 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/.tmp/c/81db6571d5254ce284092c4dda1e151f as hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/c/81db6571d5254ce284092c4dda1e151f 2024-12-09T08:19:40,045 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/c/81db6571d5254ce284092c4dda1e151f, entries=6, sequenceid=26, filesize=5.3 K 2024-12-09T08:19:40,046 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for a4d0037a191e85107548afaafaf2f7b3 in 100ms, sequenceid=26, compaction requested=false; wal=null 2024-12-09T08:19:40,047 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/recovered.edits/0000000000000000026 2024-12-09T08:19:40,048 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:40,048 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:40,048 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:19:40,050 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:40,052 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/hbase/data/default/testReplayEditsAfterAbortingFlush/a4d0037a191e85107548afaafaf2f7b3/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-09T08:19:40,053 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a4d0037a191e85107548afaafaf2f7b3; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71219831, jitterRate=0.06125818192958832}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:19:40,053 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a4d0037a191e85107548afaafaf2f7b3 2024-12-09T08:19:40,054 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a4d0037a191e85107548afaafaf2f7b3: Running coprocessor pre-open hook at 1733732379913Writing region info on filesystem at 1733732379913Initializing all the Stores at 1733732379913Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732379913Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732379914 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732379914Obtaining lock to block concurrent updates at 1733732379947 (+33 ms)Preparing flush snapshotting stores in a4d0037a191e85107548afaafaf2f7b3 at 1733732379947Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733732370552.a4d0037a191e85107548afaafaf2f7b3., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733732379947Flushing stores of testReplayEditsAfterAbortingFlush,,1733732370552.a4d0037a191e85107548afaafaf2f7b3. at 1733732379947Flushing a4d0037a191e85107548afaafaf2f7b3/a: creating writer at 1733732379947Flushing a4d0037a191e85107548afaafaf2f7b3/a: appending metadata at 1733732379959 (+12 ms)Flushing a4d0037a191e85107548afaafaf2f7b3/a: closing flushed file at 1733732379959Flushing a4d0037a191e85107548afaafaf2f7b3/b: creating writer at 1733732379972 (+13 ms)Flushing a4d0037a191e85107548afaafaf2f7b3/b: appending metadata at 1733732379984 (+12 ms)Flushing a4d0037a191e85107548afaafaf2f7b3/b: closing flushed file at 1733732379984Flushing a4d0037a191e85107548afaafaf2f7b3/c: creating writer at 1733732379995 (+11 ms)Flushing a4d0037a191e85107548afaafaf2f7b3/c: appending metadata at 1733732380009 (+14 ms)Flushing a4d0037a191e85107548afaafaf2f7b3/c: closing flushed file at 1733732380009Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e7483aa: reopening flushed file at 1733732380025 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a9e5275: reopening flushed file at 1733732380032 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b79adbe: reopening flushed file at 1733732380039 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for a4d0037a191e85107548afaafaf2f7b3 in 100ms, sequenceid=26, compaction requested=false; wal=null at 1733732380046 (+7 ms)Cleaning up temporary data from old regions at 1733732380048 (+2 ms)Running coprocessor post-open hooks at 1733732380053 (+5 ms)Region opened successfully at 1733732380054 (+1 ms) 2024-12-09T08:19:40,074 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=415 (was 417), OpenFileDescriptor=1183 (was 1127) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=229 (was 234), ProcessCount=11 (was 11), AvailableMemoryMB=4292 (was 4326) 2024-12-09T08:19:40,074 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1183 is superior to 1024 2024-12-09T08:19:40,084 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=415, OpenFileDescriptor=1183, MaxFileDescriptor=1048576, SystemLoadAverage=229, ProcessCount=11, AvailableMemoryMB=4291 2024-12-09T08:19:40,084 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1183 is superior to 1024 2024-12-09T08:19:40,099 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:40,100 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:40,101 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:19:40,103 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-71127370, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/hregion-71127370, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:40,113 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-71127370/hregion-71127370.1733732380103, exclude list is [], retry=0 2024-12-09T08:19:40,116 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:40,116 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:40,116 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:40,138 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-71127370/hregion-71127370.1733732380103 2024-12-09T08:19:40,139 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42683:42683),(127.0.0.1/127.0.0.1:41283:41283),(127.0.0.1/127.0.0.1:42925:42925)] 2024-12-09T08:19:40,139 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d15ac1dd2d8f11802975d5a528981e0b, NAME => 'testDatalossWhenInputError,,1733732380099.d15ac1dd2d8f11802975d5a528981e0b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45647/hbase 2024-12-09T08:19:40,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741910_1088 (size=61) 2024-12-09T08:19:40,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741910_1088 (size=61) 2024-12-09T08:19:40,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741910_1088 (size=61) 2024-12-09T08:19:40,148 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733732380099.d15ac1dd2d8f11802975d5a528981e0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:40,149 INFO [StoreOpener-d15ac1dd2d8f11802975d5a528981e0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,150 INFO [StoreOpener-d15ac1dd2d8f11802975d5a528981e0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d15ac1dd2d8f11802975d5a528981e0b columnFamilyName a 2024-12-09T08:19:40,150 DEBUG [StoreOpener-d15ac1dd2d8f11802975d5a528981e0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:40,150 INFO [StoreOpener-d15ac1dd2d8f11802975d5a528981e0b-1 {}] regionserver.HStore(327): Store=d15ac1dd2d8f11802975d5a528981e0b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:40,151 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,151 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,151 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,152 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,152 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,153 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,155 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:19:40,156 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d15ac1dd2d8f11802975d5a528981e0b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64762311, jitterRate=-0.0349663645029068}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T08:19:40,156 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d15ac1dd2d8f11802975d5a528981e0b: Writing region info on filesystem at 1733732380148Initializing all the Stores at 1733732380148Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732380148Cleaning up temporary data from old regions at 1733732380152 (+4 ms)Region opened successfully at 1733732380156 (+4 ms) 2024-12-09T08:19:40,156 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d15ac1dd2d8f11802975d5a528981e0b, disabling compactions & flushes 2024-12-09T08:19:40,156 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733732380099.d15ac1dd2d8f11802975d5a528981e0b. 2024-12-09T08:19:40,156 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733732380099.d15ac1dd2d8f11802975d5a528981e0b. 2024-12-09T08:19:40,156 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733732380099.d15ac1dd2d8f11802975d5a528981e0b. after waiting 0 ms 2024-12-09T08:19:40,156 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733732380099.d15ac1dd2d8f11802975d5a528981e0b. 2024-12-09T08:19:40,157 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733732380099.d15ac1dd2d8f11802975d5a528981e0b. 2024-12-09T08:19:40,157 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d15ac1dd2d8f11802975d5a528981e0b: Waiting for close lock at 1733732380156Disabling compacts and flushes for region at 1733732380156Disabling writes for close at 1733732380156Writing region close event to WAL at 1733732380157 (+1 ms)Closed at 1733732380157 2024-12-09T08:19:40,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741909_1087 (size=175) 2024-12-09T08:19:40,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741909_1087 (size=175) 2024-12-09T08:19:40,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741909_1087 (size=175) 2024-12-09T08:19:40,161 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:19:40,161 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-71127370:(num 1733732380103) 2024-12-09T08:19:40,161 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:19:40,163 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732380098, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:40,173 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733732380098/wal.1733732380163, exclude list is [], retry=0 2024-12-09T08:19:40,175 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:40,176 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:40,176 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:40,199 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733732380098/wal.1733732380163 2024-12-09T08:19:40,200 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42925:42925),(127.0.0.1/127.0.0.1:41283:41283),(127.0.0.1/127.0.0.1:42683:42683)] 2024-12-09T08:19:40,200 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d15ac1dd2d8f11802975d5a528981e0b, NAME => 'testDatalossWhenInputError,,1733732380099.d15ac1dd2d8f11802975d5a528981e0b.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:19:40,200 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733732380099.d15ac1dd2d8f11802975d5a528981e0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:40,200 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,200 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,201 INFO [StoreOpener-d15ac1dd2d8f11802975d5a528981e0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,202 INFO [StoreOpener-d15ac1dd2d8f11802975d5a528981e0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d15ac1dd2d8f11802975d5a528981e0b columnFamilyName a 2024-12-09T08:19:40,202 DEBUG [StoreOpener-d15ac1dd2d8f11802975d5a528981e0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:40,203 INFO [StoreOpener-d15ac1dd2d8f11802975d5a528981e0b-1 {}] regionserver.HStore(327): Store=d15ac1dd2d8f11802975d5a528981e0b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:40,203 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,203 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,204 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,205 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,205 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,206 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,207 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d15ac1dd2d8f11802975d5a528981e0b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73700417, jitterRate=0.0982217937707901}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T08:19:40,208 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d15ac1dd2d8f11802975d5a528981e0b: Writing region info on filesystem at 1733732380200Initializing all the Stores at 1733732380201 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732380201Cleaning up temporary data from old regions at 1733732380205 (+4 ms)Region opened successfully at 1733732380208 (+3 ms) 2024-12-09T08:19:40,216 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d15ac1dd2d8f11802975d5a528981e0b, disabling compactions & flushes 2024-12-09T08:19:40,216 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733732380099.d15ac1dd2d8f11802975d5a528981e0b. 2024-12-09T08:19:40,216 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733732380099.d15ac1dd2d8f11802975d5a528981e0b. 2024-12-09T08:19:40,216 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733732380099.d15ac1dd2d8f11802975d5a528981e0b. after waiting 0 ms 2024-12-09T08:19:40,216 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733732380099.d15ac1dd2d8f11802975d5a528981e0b. 2024-12-09T08:19:40,217 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733732380099.d15ac1dd2d8f11802975d5a528981e0b. 2024-12-09T08:19:40,217 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733732380099.d15ac1dd2d8f11802975d5a528981e0b. 2024-12-09T08:19:40,217 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d15ac1dd2d8f11802975d5a528981e0b: Waiting for close lock at 1733732380216Disabling compacts and flushes for region at 1733732380216Disabling writes for close at 1733732380216Writing region close event to WAL at 1733732380217 (+1 ms)Closed at 1733732380217 2024-12-09T08:19:40,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741911_1089 (size=1795) 2024-12-09T08:19:40,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741911_1089 (size=1795) 2024-12-09T08:19:40,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741911_1089 (size=1795) 2024-12-09T08:19:40,232 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45647/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732380098/wal.1733732380163, size=1.8 K (1795bytes) 2024-12-09T08:19:40,232 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45647/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732380098/wal.1733732380163 2024-12-09T08:19:40,232 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45647/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732380098/wal.1733732380163 after 0ms 2024-12-09T08:19:40,256 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:40,257 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45647/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732380098/wal.1733732380163 took 25ms 2024-12-09T08:19:40,278 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45647/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732380098/wal.1733732380163 so closing down 2024-12-09T08:19:40,278 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:19:40,279 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733732380163.temp 2024-12-09T08:19:40,300 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b/recovered.edits/0000000000000000003-wal.1733732380163.temp 2024-12-09T08:19:40,301 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:19:40,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741912_1090 (size=1795) 2024-12-09T08:19:40,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741912_1090 (size=1795) 2024-12-09T08:19:40,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741912_1090 (size=1795) 2024-12-09T08:19:40,307 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b/recovered.edits/0000000000000000003-wal.1733732380163.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-09T08:19:40,308 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b/recovered.edits/0000000000000000003-wal.1733732380163.temp to hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b/recovered.edits/0000000000000000012 2024-12-09T08:19:40,309 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 32 ms; skipped=0; WAL=hdfs://localhost:45647/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732380098/wal.1733732380163, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-12-09T08:19:40,309 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45647/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732380098/wal.1733732380163, journal: Splitting hdfs://localhost:45647/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732380098/wal.1733732380163, size=1.8 K (1795bytes) at 1733732380232Finishing writing output for hdfs://localhost:45647/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732380098/wal.1733732380163 so closing down at 1733732380278 (+46 ms)Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b/recovered.edits/0000000000000000003-wal.1733732380163.temp at 1733732380301 (+23 ms)3 split writer threads finished at 1733732380301Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b/recovered.edits/0000000000000000003-wal.1733732380163.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733732380307 (+6 ms)Rename recovered edits hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b/recovered.edits/0000000000000000003-wal.1733732380163.temp to hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b/recovered.edits/0000000000000000012 at 1733732380308 (+1 ms)Processed 10 edits across 1 Regions in 32 ms; skipped=0; WAL=hdfs://localhost:45647/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732380098/wal.1733732380163, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1733732380309 (+1 ms) 2024-12-09T08:19:40,310 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45647/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732380098/wal.1733732380163 to hdfs://localhost:45647/hbase/oldWALs/wal.1733732380163 2024-12-09T08:19:40,311 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b/recovered.edits/0000000000000000012 2024-12-09T08:19:40,335 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:40,615 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:19:40,617 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/testdatalosswheninputerror-manual,16010,1733732380098, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:40,628 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733732380098/wal.1733732380618, exclude list is [], retry=0 2024-12-09T08:19:40,630 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:40,631 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:40,631 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:40,653 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733732380098/wal.1733732380618 2024-12-09T08:19:40,654 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41283:41283),(127.0.0.1/127.0.0.1:42683:42683),(127.0.0.1/127.0.0.1:42925:42925)] 2024-12-09T08:19:40,654 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d15ac1dd2d8f11802975d5a528981e0b, NAME => 'testDatalossWhenInputError,,1733732380099.d15ac1dd2d8f11802975d5a528981e0b.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:19:40,654 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733732380099.d15ac1dd2d8f11802975d5a528981e0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:40,654 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,654 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,657 INFO [StoreOpener-d15ac1dd2d8f11802975d5a528981e0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,658 INFO [StoreOpener-d15ac1dd2d8f11802975d5a528981e0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d15ac1dd2d8f11802975d5a528981e0b columnFamilyName a 2024-12-09T08:19:40,658 DEBUG [StoreOpener-d15ac1dd2d8f11802975d5a528981e0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:40,659 INFO [StoreOpener-d15ac1dd2d8f11802975d5a528981e0b-1 {}] regionserver.HStore(327): Store=d15ac1dd2d8f11802975d5a528981e0b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:40,659 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,660 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,661 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,662 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b/recovered.edits/0000000000000000012 2024-12-09T08:19:40,685 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:40,686 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b/recovered.edits/0000000000000000012 2024-12-09T08:19:40,686 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d15ac1dd2d8f11802975d5a528981e0b 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-09T08:19:40,702 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b/.tmp/a/113f95fd902248e98b7c9a54fc61ba7e is 79, key is testDatalossWhenInputError/a:x0/1733732380208/Put/seqid=0 2024-12-09T08:19:40,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741914_1092 (size=5808) 2024-12-09T08:19:40,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741914_1092 (size=5808) 2024-12-09T08:19:40,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741914_1092 (size=5808) 2024-12-09T08:19:40,709 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b/.tmp/a/113f95fd902248e98b7c9a54fc61ba7e 2024-12-09T08:19:40,718 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b/.tmp/a/113f95fd902248e98b7c9a54fc61ba7e as hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b/a/113f95fd902248e98b7c9a54fc61ba7e 2024-12-09T08:19:40,726 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b/a/113f95fd902248e98b7c9a54fc61ba7e, entries=10, sequenceid=12, filesize=5.7 K 2024-12-09T08:19:40,727 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for d15ac1dd2d8f11802975d5a528981e0b in 41ms, sequenceid=12, compaction requested=false; wal=null 2024-12-09T08:19:40,727 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b/recovered.edits/0000000000000000012 2024-12-09T08:19:40,728 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,729 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,732 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,734 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-09T08:19:40,735 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d15ac1dd2d8f11802975d5a528981e0b; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68992655, jitterRate=0.02807067334651947}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T08:19:40,736 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d15ac1dd2d8f11802975d5a528981e0b: Writing region info on filesystem at 1733732380654Initializing all the Stores at 1733732380656 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732380657 (+1 ms)Obtaining lock to block concurrent updates at 1733732380686 (+29 ms)Preparing flush snapshotting stores in d15ac1dd2d8f11802975d5a528981e0b at 1733732380686Finished memstore snapshotting testDatalossWhenInputError,,1733732380099.d15ac1dd2d8f11802975d5a528981e0b., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733732380686Flushing stores of testDatalossWhenInputError,,1733732380099.d15ac1dd2d8f11802975d5a528981e0b. at 1733732380686Flushing d15ac1dd2d8f11802975d5a528981e0b/a: creating writer at 1733732380686Flushing d15ac1dd2d8f11802975d5a528981e0b/a: appending metadata at 1733732380702 (+16 ms)Flushing d15ac1dd2d8f11802975d5a528981e0b/a: closing flushed file at 1733732380702Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@387fa134: reopening flushed file at 1733732380716 (+14 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for d15ac1dd2d8f11802975d5a528981e0b in 41ms, sequenceid=12, compaction requested=false; wal=null at 1733732380727 (+11 ms)Cleaning up temporary data from old regions at 1733732380729 (+2 ms)Region opened successfully at 1733732380736 (+7 ms) 2024-12-09T08:19:40,739 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d15ac1dd2d8f11802975d5a528981e0b, NAME => 'testDatalossWhenInputError,,1733732380099.d15ac1dd2d8f11802975d5a528981e0b.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:19:40,739 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733732380099.d15ac1dd2d8f11802975d5a528981e0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:40,739 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,739 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,741 INFO [StoreOpener-d15ac1dd2d8f11802975d5a528981e0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,742 INFO [StoreOpener-d15ac1dd2d8f11802975d5a528981e0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d15ac1dd2d8f11802975d5a528981e0b columnFamilyName a 2024-12-09T08:19:40,742 DEBUG [StoreOpener-d15ac1dd2d8f11802975d5a528981e0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:40,747 DEBUG [StoreOpener-d15ac1dd2d8f11802975d5a528981e0b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b/a/113f95fd902248e98b7c9a54fc61ba7e 2024-12-09T08:19:40,747 INFO [StoreOpener-d15ac1dd2d8f11802975d5a528981e0b-1 {}] regionserver.HStore(327): Store=d15ac1dd2d8f11802975d5a528981e0b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:40,748 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,748 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,749 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,750 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,750 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,751 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d15ac1dd2d8f11802975d5a528981e0b 2024-12-09T08:19:40,753 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/hbase/data/default/testDatalossWhenInputError/d15ac1dd2d8f11802975d5a528981e0b/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-09T08:19:40,754 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d15ac1dd2d8f11802975d5a528981e0b; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72353817, jitterRate=0.07815589010715485}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T08:19:40,754 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d15ac1dd2d8f11802975d5a528981e0b: Writing region info on filesystem at 1733732380740Initializing all the Stores at 1733732380740Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732380741 (+1 ms)Cleaning up temporary data from old regions at 1733732380750 (+9 ms)Region opened successfully at 1733732380754 (+4 ms) 2024-12-09T08:19:40,767 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=424 (was 415) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:51148 [Receiving block BP-1712704723-172.17.0.2-1733732346335:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:36180 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1712704723-172.17.0.2-1733732346335:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:53496 [Receiving block BP-1712704723-172.17.0.2-1733732346335:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1712704723-172.17.0.2-1733732346335:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1712704723-172.17.0.2-1733732346335:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:53460 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:49274 [Receiving block BP-1712704723-172.17.0.2-1733732346335:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:51082 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1183) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=229 (was 229), ProcessCount=11 (was 11), AvailableMemoryMB=4265 (was 4291) 2024-12-09T08:19:40,768 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-09T08:19:40,778 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=424, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=229, ProcessCount=11, AvailableMemoryMB=4264 2024-12-09T08:19:40,778 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-09T08:19:40,791 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:40,793 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:40,793 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:19:40,795 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-70752940, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/hregion-70752940, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:40,805 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-70752940/hregion-70752940.1733732380796, exclude list is [], retry=0 2024-12-09T08:19:40,808 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:40,808 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:40,808 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:40,828 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-70752940/hregion-70752940.1733732380796 2024-12-09T08:19:40,828 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42925:42925),(127.0.0.1/127.0.0.1:42683:42683),(127.0.0.1/127.0.0.1:41283:41283)] 2024-12-09T08:19:40,828 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 00833a8a8dbfff32f7c4201a5346a191, NAME => 'testCompactedBulkLoadedFiles,,1733732380792.00833a8a8dbfff32f7c4201a5346a191.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45647/hbase 2024-12-09T08:19:40,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741916_1094 (size=63) 2024-12-09T08:19:40,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741916_1094 (size=63) 2024-12-09T08:19:40,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741916_1094 (size=63) 2024-12-09T08:19:40,837 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733732380792.00833a8a8dbfff32f7c4201a5346a191.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:40,838 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:40,839 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00833a8a8dbfff32f7c4201a5346a191 columnFamilyName a 2024-12-09T08:19:40,839 DEBUG [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:40,840 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.HStore(327): Store=00833a8a8dbfff32f7c4201a5346a191/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:40,840 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:40,841 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00833a8a8dbfff32f7c4201a5346a191 columnFamilyName b 2024-12-09T08:19:40,841 DEBUG [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:40,841 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.HStore(327): Store=00833a8a8dbfff32f7c4201a5346a191/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:40,841 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:40,842 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00833a8a8dbfff32f7c4201a5346a191 columnFamilyName c 2024-12-09T08:19:40,842 DEBUG [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:40,843 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.HStore(327): Store=00833a8a8dbfff32f7c4201a5346a191/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:40,843 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:40,844 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:40,844 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:40,845 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:40,845 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:40,846 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:19:40,847 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:40,849 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:19:40,849 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 00833a8a8dbfff32f7c4201a5346a191; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73370279, jitterRate=0.09330235421657562}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:19:40,850 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 00833a8a8dbfff32f7c4201a5346a191: Writing region info on filesystem at 1733732380837Initializing all the Stores at 1733732380838 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732380838Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732380838Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732380838Cleaning up temporary data from old regions at 1733732380845 (+7 ms)Region opened successfully at 1733732380850 (+5 ms) 2024-12-09T08:19:40,850 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 00833a8a8dbfff32f7c4201a5346a191, disabling compactions & flushes 2024-12-09T08:19:40,850 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733732380792.00833a8a8dbfff32f7c4201a5346a191. 2024-12-09T08:19:40,850 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733732380792.00833a8a8dbfff32f7c4201a5346a191. 2024-12-09T08:19:40,850 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733732380792.00833a8a8dbfff32f7c4201a5346a191. after waiting 0 ms 2024-12-09T08:19:40,850 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733732380792.00833a8a8dbfff32f7c4201a5346a191. 2024-12-09T08:19:40,850 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733732380792.00833a8a8dbfff32f7c4201a5346a191. 2024-12-09T08:19:40,850 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 00833a8a8dbfff32f7c4201a5346a191: Waiting for close lock at 1733732380850Disabling compacts and flushes for region at 1733732380850Disabling writes for close at 1733732380850Writing region close event to WAL at 1733732380850Closed at 1733732380850 2024-12-09T08:19:40,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741915_1093 (size=175) 2024-12-09T08:19:40,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741915_1093 (size=175) 2024-12-09T08:19:40,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741915_1093 (size=175) 2024-12-09T08:19:40,855 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:19:40,855 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-70752940:(num 1733732380796) 2024-12-09T08:19:40,855 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:19:40,857 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732380791, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:40,867 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732380791/wal.1733732380857, exclude list is [], retry=0 2024-12-09T08:19:40,870 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:40,870 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:40,870 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:40,893 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732380791/wal.1733732380857 2024-12-09T08:19:40,893 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41283:41283),(127.0.0.1/127.0.0.1:42683:42683),(127.0.0.1/127.0.0.1:42925:42925)] 2024-12-09T08:19:40,893 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 00833a8a8dbfff32f7c4201a5346a191, NAME => 'testCompactedBulkLoadedFiles,,1733732380792.00833a8a8dbfff32f7c4201a5346a191.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:19:40,894 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733732380792.00833a8a8dbfff32f7c4201a5346a191.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:40,894 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:40,894 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:40,895 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:40,895 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00833a8a8dbfff32f7c4201a5346a191 columnFamilyName a 2024-12-09T08:19:40,896 DEBUG [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:40,896 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.HStore(327): Store=00833a8a8dbfff32f7c4201a5346a191/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:40,896 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:40,897 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00833a8a8dbfff32f7c4201a5346a191 columnFamilyName b 2024-12-09T08:19:40,897 DEBUG [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:40,897 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.HStore(327): Store=00833a8a8dbfff32f7c4201a5346a191/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:40,897 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:40,898 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00833a8a8dbfff32f7c4201a5346a191 columnFamilyName c 2024-12-09T08:19:40,898 DEBUG [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:40,898 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.HStore(327): Store=00833a8a8dbfff32f7c4201a5346a191/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:40,898 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:40,899 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:40,900 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:40,901 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:40,901 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:40,901 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:19:40,903 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:40,904 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 00833a8a8dbfff32f7c4201a5346a191; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68179644, jitterRate=0.015955865383148193}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:19:40,905 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 00833a8a8dbfff32f7c4201a5346a191: Writing region info on filesystem at 1733732380894Initializing all the Stores at 1733732380895 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732380895Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732380895Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732380895Cleaning up temporary data from old regions at 1733732380901 (+6 ms)Region opened successfully at 1733732380905 (+4 ms) 2024-12-09T08:19:40,909 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733732380908/Put/seqid=0 2024-12-09T08:19:40,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741918_1096 (size=4875) 2024-12-09T08:19:40,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741918_1096 (size=4875) 2024-12-09T08:19:40,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741918_1096 (size=4875) 2024-12-09T08:19:40,917 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733732380916/Put/seqid=0 2024-12-09T08:19:40,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741919_1097 (size=4875) 2024-12-09T08:19:40,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741919_1097 (size=4875) 2024-12-09T08:19:40,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741919_1097 (size=4875) 2024-12-09T08:19:40,928 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733732380927/Put/seqid=0 2024-12-09T08:19:40,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741920_1098 (size=4875) 2024-12-09T08:19:40,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741920_1098 (size=4875) 2024-12-09T08:19:40,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741920_1098 (size=4875) 2024-12-09T08:19:40,934 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45647/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 00833a8a8dbfff32f7c4201a5346a191/a 2024-12-09T08:19:40,937 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-09T08:19:40,937 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T08:19:40,937 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45647/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 00833a8a8dbfff32f7c4201a5346a191/a 2024-12-09T08:19:40,940 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-09T08:19:40,940 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T08:19:40,940 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45647/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 00833a8a8dbfff32f7c4201a5346a191/a 2024-12-09T08:19:40,943 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-09T08:19:40,943 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T08:19:40,943 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 00833a8a8dbfff32f7c4201a5346a191 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-09T08:19:40,958 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/.tmp/a/914dd6bd0928485fbabadf43a1812a7d is 55, key is testCompactedBulkLoadedFiles/a:a/1733732380905/Put/seqid=0 2024-12-09T08:19:40,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741921_1099 (size=5107) 2024-12-09T08:19:40,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741921_1099 (size=5107) 2024-12-09T08:19:40,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741921_1099 (size=5107) 2024-12-09T08:19:40,964 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/.tmp/a/914dd6bd0928485fbabadf43a1812a7d 2024-12-09T08:19:40,970 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/.tmp/a/914dd6bd0928485fbabadf43a1812a7d as hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/914dd6bd0928485fbabadf43a1812a7d 2024-12-09T08:19:40,975 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/914dd6bd0928485fbabadf43a1812a7d, entries=1, sequenceid=4, filesize=5.0 K 2024-12-09T08:19:40,976 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 00833a8a8dbfff32f7c4201a5346a191 in 33ms, sequenceid=4, compaction requested=false 2024-12-09T08:19:40,976 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 00833a8a8dbfff32f7c4201a5346a191: 2024-12-09T08:19:40,977 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/1baabf1b3c564cdbac1eafc00d99b95e_SeqId_4_ 2024-12-09T08:19:40,978 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/a1e3ac249e6f412880416239eed2dc47_SeqId_4_ 2024-12-09T08:19:40,979 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/c0738a45a2df482191f6750abcb9b25a_SeqId_4_ 2024-12-09T08:19:40,980 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45647/hbase/testCompactedBulkLoadedFiles/hfile0 into 00833a8a8dbfff32f7c4201a5346a191/a as hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/1baabf1b3c564cdbac1eafc00d99b95e_SeqId_4_ - updating store file list. 2024-12-09T08:19:40,984 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1baabf1b3c564cdbac1eafc00d99b95e_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T08:19:40,984 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/1baabf1b3c564cdbac1eafc00d99b95e_SeqId_4_ into 00833a8a8dbfff32f7c4201a5346a191/a 2024-12-09T08:19:40,984 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45647/hbase/testCompactedBulkLoadedFiles/hfile0 into 00833a8a8dbfff32f7c4201a5346a191/a (new location: hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/1baabf1b3c564cdbac1eafc00d99b95e_SeqId_4_) 2024-12-09T08:19:40,985 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45647/hbase/testCompactedBulkLoadedFiles/hfile1 into 00833a8a8dbfff32f7c4201a5346a191/a as hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/a1e3ac249e6f412880416239eed2dc47_SeqId_4_ - updating store file list. 2024-12-09T08:19:40,988 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for a1e3ac249e6f412880416239eed2dc47_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T08:19:40,988 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/a1e3ac249e6f412880416239eed2dc47_SeqId_4_ into 00833a8a8dbfff32f7c4201a5346a191/a 2024-12-09T08:19:40,989 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45647/hbase/testCompactedBulkLoadedFiles/hfile1 into 00833a8a8dbfff32f7c4201a5346a191/a (new location: hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/a1e3ac249e6f412880416239eed2dc47_SeqId_4_) 2024-12-09T08:19:40,990 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45647/hbase/testCompactedBulkLoadedFiles/hfile2 into 00833a8a8dbfff32f7c4201a5346a191/a as hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/c0738a45a2df482191f6750abcb9b25a_SeqId_4_ - updating store file list. 2024-12-09T08:19:40,993 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for c0738a45a2df482191f6750abcb9b25a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T08:19:40,993 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/c0738a45a2df482191f6750abcb9b25a_SeqId_4_ into 00833a8a8dbfff32f7c4201a5346a191/a 2024-12-09T08:19:40,993 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45647/hbase/testCompactedBulkLoadedFiles/hfile2 into 00833a8a8dbfff32f7c4201a5346a191/a (new location: hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/c0738a45a2df482191f6750abcb9b25a_SeqId_4_) 2024-12-09T08:19:41,001 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T08:19:41,001 DEBUG [Time-limited test {}] regionserver.HStore(1541): 00833a8a8dbfff32f7c4201a5346a191/a is initiating major compaction (all files) 2024-12-09T08:19:41,001 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 00833a8a8dbfff32f7c4201a5346a191/a in testCompactedBulkLoadedFiles,,1733732380792.00833a8a8dbfff32f7c4201a5346a191. 2024-12-09T08:19:41,001 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/914dd6bd0928485fbabadf43a1812a7d, hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/1baabf1b3c564cdbac1eafc00d99b95e_SeqId_4_, hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/a1e3ac249e6f412880416239eed2dc47_SeqId_4_, hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/c0738a45a2df482191f6750abcb9b25a_SeqId_4_] into tmpdir=hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/.tmp, totalSize=19.3 K 2024-12-09T08:19:41,002 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 914dd6bd0928485fbabadf43a1812a7d, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733732380905 2024-12-09T08:19:41,002 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1baabf1b3c564cdbac1eafc00d99b95e_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-09T08:19:41,002 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting a1e3ac249e6f412880416239eed2dc47_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-09T08:19:41,003 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c0738a45a2df482191f6750abcb9b25a_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-09T08:19:41,015 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/.tmp/a/2357a0007db74342ae7601163f02de5e is 55, key is testCompactedBulkLoadedFiles/a:a/1733732380905/Put/seqid=0 2024-12-09T08:19:41,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741922_1100 (size=6154) 2024-12-09T08:19:41,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741922_1100 (size=6154) 2024-12-09T08:19:41,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741922_1100 (size=6154) 2024-12-09T08:19:41,026 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/.tmp/a/2357a0007db74342ae7601163f02de5e as hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/2357a0007db74342ae7601163f02de5e 2024-12-09T08:19:41,032 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 00833a8a8dbfff32f7c4201a5346a191/a of 00833a8a8dbfff32f7c4201a5346a191 into 2357a0007db74342ae7601163f02de5e(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T08:19:41,032 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 00833a8a8dbfff32f7c4201a5346a191: 2024-12-09T08:19:41,032 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-09T08:19:41,033 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-09T08:19:41,066 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-09T08:19:41,067 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-09T08:19:41,067 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-09T08:19:41,068 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-09T08:19:41,069 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732380791/wal.1733732380857, size=0 (0bytes) 2024-12-09T08:19:41,069 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:45647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732380791/wal.1733732380857 might be still open, length is 0 2024-12-09T08:19:41,069 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732380791/wal.1733732380857 2024-12-09T08:19:41,070 WARN [IPC Server handler 3 on default port 45647 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732380791/wal.1733732380857 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-12-09T08:19:41,070 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732380791/wal.1733732380857 after 1ms 2024-12-09T08:19:42,926 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:53528 [Receiving block BP-1712704723-172.17.0.2-1733732346335:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:45463:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53528 dst: /127.0.0.1:45463 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45463 remote=/127.0.0.1:53528]. Total timeout mills is 60000, 58107 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:42,926 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:51188 [Receiving block BP-1712704723-172.17.0.2-1733732346335:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:41399:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51188 dst: /127.0.0.1:41399 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:42,926 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:49316 [Receiving block BP-1712704723-172.17.0.2-1733732346335:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:38689:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49316 dst: /127.0.0.1:38689 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:42,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741917_1101 (size=1794) 2024-12-09T08:19:42,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741917_1101 (size=1794) 2024-12-09T08:19:45,071 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732380791/wal.1733732380857 after 4002ms 2024-12-09T08:19:45,105 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:45,105 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732380791/wal.1733732380857 took 4036ms 2024-12-09T08:19:45,107 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:45647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732380791/wal.1733732380857; continuing. 2024-12-09T08:19:45,107 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732380791/wal.1733732380857 so closing down 2024-12-09T08:19:45,107 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:19:45,109 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733732380857.temp 2024-12-09T08:19:45,130 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/recovered.edits/0000000000000000003-wal.1733732380857.temp 2024-12-09T08:19:45,130 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:19:45,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741923_1102 (size=800) 2024-12-09T08:19:45,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741923_1102 (size=800) 2024-12-09T08:19:45,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741923_1102 (size=800) 2024-12-09T08:19:45,136 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/recovered.edits/0000000000000000003-wal.1733732380857.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-09T08:19:45,137 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/recovered.edits/0000000000000000003-wal.1733732380857.temp to hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/recovered.edits/0000000000000000008 2024-12-09T08:19:45,137 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 31 ms; skipped=3; WAL=hdfs://localhost:45647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732380791/wal.1733732380857, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T08:19:45,137 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732380791/wal.1733732380857, journal: Splitting hdfs://localhost:45647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732380791/wal.1733732380857, size=0 (0bytes) at 1733732381069Finishing writing output for hdfs://localhost:45647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732380791/wal.1733732380857 so closing down at 1733732385107 (+4038 ms)Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/recovered.edits/0000000000000000003-wal.1733732380857.temp at 1733732385130 (+23 ms)3 split writer threads finished at 1733732385130Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/recovered.edits/0000000000000000003-wal.1733732380857.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733732385136 (+6 ms)Rename recovered edits hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/recovered.edits/0000000000000000003-wal.1733732380857.temp to hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/recovered.edits/0000000000000000008 at 1733732385137 (+1 ms)Processed 5 edits across 1 Regions in 31 ms; skipped=3; WAL=hdfs://localhost:45647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732380791/wal.1733732380857, size=0, length=0, corrupted=false, cancelled=false at 1733732385137 2024-12-09T08:19:45,138 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732380791/wal.1733732380857 to hdfs://localhost:45647/hbase/oldWALs/wal.1733732380857 2024-12-09T08:19:45,139 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/recovered.edits/0000000000000000008 2024-12-09T08:19:45,139 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:19:45,140 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732380791, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:45,152 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732380791/wal.1733732385141, exclude list is [], retry=0 2024-12-09T08:19:45,154 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:45,155 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:45,155 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:45,177 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732380791/wal.1733732385141 2024-12-09T08:19:45,177 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42925:42925),(127.0.0.1/127.0.0.1:41283:41283),(127.0.0.1/127.0.0.1:42683:42683)] 2024-12-09T08:19:45,177 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 00833a8a8dbfff32f7c4201a5346a191, NAME => 'testCompactedBulkLoadedFiles,,1733732380792.00833a8a8dbfff32f7c4201a5346a191.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:19:45,177 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733732380792.00833a8a8dbfff32f7c4201a5346a191.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:45,177 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:45,177 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:45,179 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:45,179 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00833a8a8dbfff32f7c4201a5346a191 columnFamilyName a 2024-12-09T08:19:45,179 DEBUG [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:45,185 DEBUG [StoreFileOpener-00833a8a8dbfff32f7c4201a5346a191-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1baabf1b3c564cdbac1eafc00d99b95e_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T08:19:45,185 DEBUG [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/1baabf1b3c564cdbac1eafc00d99b95e_SeqId_4_ 2024-12-09T08:19:45,189 DEBUG [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/2357a0007db74342ae7601163f02de5e 2024-12-09T08:19:45,192 DEBUG [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/914dd6bd0928485fbabadf43a1812a7d 2024-12-09T08:19:45,195 DEBUG [StoreFileOpener-00833a8a8dbfff32f7c4201a5346a191-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for a1e3ac249e6f412880416239eed2dc47_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T08:19:45,195 DEBUG [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/a1e3ac249e6f412880416239eed2dc47_SeqId_4_ 2024-12-09T08:19:45,198 DEBUG [StoreFileOpener-00833a8a8dbfff32f7c4201a5346a191-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for c0738a45a2df482191f6750abcb9b25a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T08:19:45,198 DEBUG [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/c0738a45a2df482191f6750abcb9b25a_SeqId_4_ 2024-12-09T08:19:45,198 WARN [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/1baabf1b3c564cdbac1eafc00d99b95e_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@2f6e9d85 2024-12-09T08:19:45,199 WARN [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/914dd6bd0928485fbabadf43a1812a7d from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@2f6e9d85 2024-12-09T08:19:45,199 WARN [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/a1e3ac249e6f412880416239eed2dc47_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@2f6e9d85 2024-12-09T08:19:45,199 WARN [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/c0738a45a2df482191f6750abcb9b25a_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@2f6e9d85 2024-12-09T08:19:45,199 DEBUG [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/1baabf1b3c564cdbac1eafc00d99b95e_SeqId_4_, hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/914dd6bd0928485fbabadf43a1812a7d, hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/a1e3ac249e6f412880416239eed2dc47_SeqId_4_, hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/c0738a45a2df482191f6750abcb9b25a_SeqId_4_] to archive 2024-12-09T08:19:45,199 DEBUG [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T08:19:45,201 DEBUG [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/1baabf1b3c564cdbac1eafc00d99b95e_SeqId_4_ to hdfs://localhost:45647/hbase/archive/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/1baabf1b3c564cdbac1eafc00d99b95e_SeqId_4_ 2024-12-09T08:19:45,202 DEBUG [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/914dd6bd0928485fbabadf43a1812a7d to hdfs://localhost:45647/hbase/archive/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/914dd6bd0928485fbabadf43a1812a7d 2024-12-09T08:19:45,203 DEBUG [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/a1e3ac249e6f412880416239eed2dc47_SeqId_4_ to hdfs://localhost:45647/hbase/archive/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/a1e3ac249e6f412880416239eed2dc47_SeqId_4_ 2024-12-09T08:19:45,204 DEBUG [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/c0738a45a2df482191f6750abcb9b25a_SeqId_4_ to hdfs://localhost:45647/hbase/archive/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/a/c0738a45a2df482191f6750abcb9b25a_SeqId_4_ 2024-12-09T08:19:45,204 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.HStore(327): Store=00833a8a8dbfff32f7c4201a5346a191/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:45,204 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:45,205 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00833a8a8dbfff32f7c4201a5346a191 columnFamilyName b 2024-12-09T08:19:45,205 DEBUG [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:45,205 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.HStore(327): Store=00833a8a8dbfff32f7c4201a5346a191/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:45,205 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:45,206 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00833a8a8dbfff32f7c4201a5346a191 columnFamilyName c 2024-12-09T08:19:45,206 DEBUG [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:45,206 INFO [StoreOpener-00833a8a8dbfff32f7c4201a5346a191-1 {}] regionserver.HStore(327): Store=00833a8a8dbfff32f7c4201a5346a191/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:45,206 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:45,207 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:45,208 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:45,208 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/recovered.edits/0000000000000000008 2024-12-09T08:19:45,229 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:45,230 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 00833a8a8dbfff32f7c4201a5346a191 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "00833a8a8dbfff32f7c4201a5346a191" family_name: "a" compaction_input: "914dd6bd0928485fbabadf43a1812a7d" compaction_input: "1baabf1b3c564cdbac1eafc00d99b95e_SeqId_4_" compaction_input: "a1e3ac249e6f412880416239eed2dc47_SeqId_4_" compaction_input: "c0738a45a2df482191f6750abcb9b25a_SeqId_4_" compaction_output: "2357a0007db74342ae7601163f02de5e" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733732380792.00833a8a8dbfff32f7c4201a5346a191." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-09T08:19:45,230 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-09T08:19:45,231 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/recovered.edits/0000000000000000008 2024-12-09T08:19:45,231 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/recovered.edits/0000000000000000008 2024-12-09T08:19:45,232 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:45,232 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:45,233 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:19:45,234 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 00833a8a8dbfff32f7c4201a5346a191 2024-12-09T08:19:45,236 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/hbase/data/default/testCompactedBulkLoadedFiles/00833a8a8dbfff32f7c4201a5346a191/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-09T08:19:45,237 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 00833a8a8dbfff32f7c4201a5346a191; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73803544, jitterRate=0.09975850582122803}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:19:45,237 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 00833a8a8dbfff32f7c4201a5346a191: Writing region info on filesystem at 1733732385177Initializing all the Stores at 1733732385178 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732385178Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732385178Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732385178Cleaning up temporary data from old regions at 1733732385232 (+54 ms)Region opened successfully at 1733732385237 (+5 ms) 2024-12-09T08:19:45,239 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 00833a8a8dbfff32f7c4201a5346a191, disabling compactions & flushes 2024-12-09T08:19:45,239 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733732380792.00833a8a8dbfff32f7c4201a5346a191. 2024-12-09T08:19:45,239 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733732380792.00833a8a8dbfff32f7c4201a5346a191. 2024-12-09T08:19:45,239 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733732380792.00833a8a8dbfff32f7c4201a5346a191. after waiting 0 ms 2024-12-09T08:19:45,239 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733732380792.00833a8a8dbfff32f7c4201a5346a191. 2024-12-09T08:19:45,240 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733732380792.00833a8a8dbfff32f7c4201a5346a191. 2024-12-09T08:19:45,240 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 00833a8a8dbfff32f7c4201a5346a191: Waiting for close lock at 1733732385239Disabling compacts and flushes for region at 1733732385239Disabling writes for close at 1733732385239Writing region close event to WAL at 1733732385240 (+1 ms)Closed at 1733732385240 2024-12-09T08:19:45,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741924_1103 (size=175) 2024-12-09T08:19:45,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741924_1103 (size=175) 2024-12-09T08:19:45,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741924_1103 (size=175) 2024-12-09T08:19:45,244 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:19:45,244 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733732385141) 2024-12-09T08:19:45,257 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=438 (was 424) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_271492737_22 at /127.0.0.1:49364 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:45647 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_271492737_22 at /127.0.0.1:53626 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_271492737_22 at /127.0.0.1:51232 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:45647 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1346 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=251 (was 229) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4209 (was 4264) 2024-12-09T08:19:45,257 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-12-09T08:19:45,268 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=438, OpenFileDescriptor=1346, MaxFileDescriptor=1048576, SystemLoadAverage=251, ProcessCount=11, AvailableMemoryMB=4208 2024-12-09T08:19:45,268 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-12-09T08:19:45,279 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:45,280 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T08:19:45,281 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T08:19:45,283 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-11495868, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/hregion-11495868, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:45,293 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-11495868/hregion-11495868.1733732385283, exclude list is [], retry=0 2024-12-09T08:19:45,296 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:45,296 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:45,296 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:45,317 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-11495868/hregion-11495868.1733732385283 2024-12-09T08:19:45,318 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42683:42683),(127.0.0.1/127.0.0.1:41283:41283),(127.0.0.1/127.0.0.1:42925:42925)] 2024-12-09T08:19:45,318 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8d2477d17af6e74ff8cc1d16184fd888, NAME => 'testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45647/hbase 2024-12-09T08:19:45,321 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T08:19:45,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741926_1105 (size=67) 2024-12-09T08:19:45,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741926_1105 (size=67) 2024-12-09T08:19:45,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741926_1105 (size=67) 2024-12-09T08:19:45,329 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:45,330 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,331 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d2477d17af6e74ff8cc1d16184fd888 columnFamilyName a 2024-12-09T08:19:45,331 DEBUG [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:45,332 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.HStore(327): Store=8d2477d17af6e74ff8cc1d16184fd888/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:45,332 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,333 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d2477d17af6e74ff8cc1d16184fd888 columnFamilyName b 2024-12-09T08:19:45,333 DEBUG [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:45,334 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.HStore(327): Store=8d2477d17af6e74ff8cc1d16184fd888/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:45,334 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,335 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d2477d17af6e74ff8cc1d16184fd888 columnFamilyName c 2024-12-09T08:19:45,335 DEBUG [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:45,336 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.HStore(327): Store=8d2477d17af6e74ff8cc1d16184fd888/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:45,336 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,337 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,337 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,339 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,339 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,339 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:19:45,340 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,343 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T08:19:45,344 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8d2477d17af6e74ff8cc1d16184fd888; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60539470, jitterRate=-0.09789159893989563}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:19:45,344 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8d2477d17af6e74ff8cc1d16184fd888: Writing region info on filesystem at 1733732385329Initializing all the Stores at 1733732385330 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732385330Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732385330Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732385330Cleaning up temporary data from old regions at 1733732385339 (+9 ms)Region opened successfully at 1733732385344 (+5 ms) 2024-12-09T08:19:45,344 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8d2477d17af6e74ff8cc1d16184fd888, disabling compactions & flushes 2024-12-09T08:19:45,344 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888. 2024-12-09T08:19:45,344 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888. 2024-12-09T08:19:45,344 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888. after waiting 0 ms 2024-12-09T08:19:45,344 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888. 2024-12-09T08:19:45,344 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888. 2024-12-09T08:19:45,345 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8d2477d17af6e74ff8cc1d16184fd888: Waiting for close lock at 1733732385344Disabling compacts and flushes for region at 1733732385344Disabling writes for close at 1733732385344Writing region close event to WAL at 1733732385344Closed at 1733732385344 2024-12-09T08:19:45,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741925_1104 (size=175) 2024-12-09T08:19:45,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741925_1104 (size=175) 2024-12-09T08:19:45,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741925_1104 (size=175) 2024-12-09T08:19:45,349 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:19:45,349 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-11495868:(num 1733732385283) 2024-12-09T08:19:45,350 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:19:45,351 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:45,363 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385352, exclude list is [], retry=0 2024-12-09T08:19:45,366 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:45,366 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:45,366 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:45,388 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385352 2024-12-09T08:19:45,388 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42925:42925),(127.0.0.1/127.0.0.1:42683:42683),(127.0.0.1/127.0.0.1:41283:41283)] 2024-12-09T08:19:45,388 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8d2477d17af6e74ff8cc1d16184fd888, NAME => 'testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:19:45,388 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:45,388 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,388 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,390 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,390 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d2477d17af6e74ff8cc1d16184fd888 columnFamilyName a 2024-12-09T08:19:45,390 DEBUG [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:45,391 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.HStore(327): Store=8d2477d17af6e74ff8cc1d16184fd888/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:45,391 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,391 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d2477d17af6e74ff8cc1d16184fd888 columnFamilyName b 2024-12-09T08:19:45,391 DEBUG [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:45,392 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.HStore(327): Store=8d2477d17af6e74ff8cc1d16184fd888/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:45,392 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,393 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d2477d17af6e74ff8cc1d16184fd888 columnFamilyName c 2024-12-09T08:19:45,393 DEBUG [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:45,393 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.HStore(327): Store=8d2477d17af6e74ff8cc1d16184fd888/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:45,393 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,394 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,395 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,396 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,396 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,396 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:19:45,397 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,398 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8d2477d17af6e74ff8cc1d16184fd888; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66589800, jitterRate=-0.00773465633392334}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:19:45,398 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8d2477d17af6e74ff8cc1d16184fd888: Writing region info on filesystem at 1733732385388Initializing all the Stores at 1733732385389 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732385389Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732385389Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732385389Cleaning up temporary data from old regions at 1733732385396 (+7 ms)Region opened successfully at 1733732385398 (+2 ms) 2024-12-09T08:19:45,405 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8d2477d17af6e74ff8cc1d16184fd888 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-09T08:19:45,420 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/.tmp/a/92cdf36c8b5c42ffafb0902083ebd460 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733732385398/Put/seqid=0 2024-12-09T08:19:45,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741928_1107 (size=5958) 2024-12-09T08:19:45,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741928_1107 (size=5958) 2024-12-09T08:19:45,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741928_1107 (size=5958) 2024-12-09T08:19:45,433 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/.tmp/a/92cdf36c8b5c42ffafb0902083ebd460 2024-12-09T08:19:45,440 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/.tmp/a/92cdf36c8b5c42ffafb0902083ebd460 as hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/a/92cdf36c8b5c42ffafb0902083ebd460 2024-12-09T08:19:45,446 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/a/92cdf36c8b5c42ffafb0902083ebd460, entries=10, sequenceid=13, filesize=5.8 K 2024-12-09T08:19:45,448 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 8d2477d17af6e74ff8cc1d16184fd888 in 43ms, sequenceid=13, compaction requested=false 2024-12-09T08:19:45,448 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8d2477d17af6e74ff8cc1d16184fd888: 2024-12-09T08:19:45,468 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8d2477d17af6e74ff8cc1d16184fd888, disabling compactions & flushes 2024-12-09T08:19:45,468 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888. 2024-12-09T08:19:45,468 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888. 2024-12-09T08:19:45,469 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888. after waiting 0 ms 2024-12-09T08:19:45,469 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888. 2024-12-09T08:19:45,469 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888. 2024-12-09T08:19:45,470 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888. 2024-12-09T08:19:45,470 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8d2477d17af6e74ff8cc1d16184fd888: Waiting for close lock at 1733732385468Disabling compacts and flushes for region at 1733732385468Disabling writes for close at 1733732385469 (+1 ms)Writing region close event to WAL at 1733732385469Closed at 1733732385469 2024-12-09T08:19:45,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741927_1106 (size=6243) 2024-12-09T08:19:45,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741927_1106 (size=6243) 2024-12-09T08:19:45,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741927_1106 (size=6243) 2024-12-09T08:19:45,493 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385352, size=6.1 K (6243bytes) 2024-12-09T08:19:45,493 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385352 2024-12-09T08:19:45,494 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385352 after 1ms 2024-12-09T08:19:45,516 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:45,516 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385352 took 23ms 2024-12-09T08:19:45,519 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385352 so closing down 2024-12-09T08:19:45,519 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:19:45,520 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733732385352.temp 2024-12-09T08:19:45,542 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/0000000000000000003-wal.1733732385352.temp 2024-12-09T08:19:45,542 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:19:45,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741929_1108 (size=5575) 2024-12-09T08:19:45,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741929_1108 (size=5575) 2024-12-09T08:19:45,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741929_1108 (size=5575) 2024-12-09T08:19:45,548 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/0000000000000000003-wal.1733732385352.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-09T08:19:45,549 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/0000000000000000003-wal.1733732385352.temp to hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/0000000000000000035 2024-12-09T08:19:45,550 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 33 ms; skipped=2; WAL=hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385352, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-12-09T08:19:45,550 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385352, journal: Splitting hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385352, size=6.1 K (6243bytes) at 1733732385493Finishing writing output for hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385352 so closing down at 1733732385519 (+26 ms)Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/0000000000000000003-wal.1733732385352.temp at 1733732385542 (+23 ms)3 split writer threads finished at 1733732385542Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/0000000000000000003-wal.1733732385352.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733732385548 (+6 ms)Rename recovered edits hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/0000000000000000003-wal.1733732385352.temp to hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/0000000000000000035 at 1733732385550 (+2 ms)Processed 32 edits across 1 Regions in 33 ms; skipped=2; WAL=hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385352, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1733732385550 2024-12-09T08:19:45,551 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385352 to hdfs://localhost:45647/hbase/oldWALs/wal.1733732385352 2024-12-09T08:19:45,557 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/0000000000000000035 2024-12-09T08:19:45,557 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:19:45,559 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:45,572 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385559, exclude list is [], retry=0 2024-12-09T08:19:45,574 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:45,575 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:45,575 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:45,597 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385559 2024-12-09T08:19:45,597 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42925:42925),(127.0.0.1/127.0.0.1:42683:42683),(127.0.0.1/127.0.0.1:41283:41283)] 2024-12-09T08:19:45,597 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8d2477d17af6e74ff8cc1d16184fd888, NAME => 'testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888.', STARTKEY => '', ENDKEY => ''} 2024-12-09T08:19:45,597 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:45,597 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,597 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,599 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,599 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d2477d17af6e74ff8cc1d16184fd888 columnFamilyName a 2024-12-09T08:19:45,599 DEBUG [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:45,604 DEBUG [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/a/92cdf36c8b5c42ffafb0902083ebd460 2024-12-09T08:19:45,604 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.HStore(327): Store=8d2477d17af6e74ff8cc1d16184fd888/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:45,604 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,605 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d2477d17af6e74ff8cc1d16184fd888 columnFamilyName b 2024-12-09T08:19:45,605 DEBUG [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:45,605 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.HStore(327): Store=8d2477d17af6e74ff8cc1d16184fd888/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:45,605 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,606 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d2477d17af6e74ff8cc1d16184fd888 columnFamilyName c 2024-12-09T08:19:45,606 DEBUG [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:45,607 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.HStore(327): Store=8d2477d17af6e74ff8cc1d16184fd888/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:45,607 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,607 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,609 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,610 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/0000000000000000035 2024-12-09T08:19:45,633 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:45,634 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/0000000000000000035 2024-12-09T08:19:45,634 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8d2477d17af6e74ff8cc1d16184fd888 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-09T08:19:45,646 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/.tmp/b/6913c5b39f21464685f62de02f345547 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733732385448/Put/seqid=0 2024-12-09T08:19:45,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741931_1110 (size=5958) 2024-12-09T08:19:45,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741931_1110 (size=5958) 2024-12-09T08:19:45,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741931_1110 (size=5958) 2024-12-09T08:19:45,653 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/.tmp/b/6913c5b39f21464685f62de02f345547 2024-12-09T08:19:45,672 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/.tmp/c/f0aa0671f657453daa54867df8bb9cf4 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733732385454/Put/seqid=0 2024-12-09T08:19:45,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741932_1111 (size=5958) 2024-12-09T08:19:45,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741932_1111 (size=5958) 2024-12-09T08:19:45,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741932_1111 (size=5958) 2024-12-09T08:19:45,678 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/.tmp/c/f0aa0671f657453daa54867df8bb9cf4 2024-12-09T08:19:45,682 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/.tmp/b/6913c5b39f21464685f62de02f345547 as hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/b/6913c5b39f21464685f62de02f345547 2024-12-09T08:19:45,688 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/b/6913c5b39f21464685f62de02f345547, entries=10, sequenceid=35, filesize=5.8 K 2024-12-09T08:19:45,689 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/.tmp/c/f0aa0671f657453daa54867df8bb9cf4 as hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/c/f0aa0671f657453daa54867df8bb9cf4 2024-12-09T08:19:45,695 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/c/f0aa0671f657453daa54867df8bb9cf4, entries=10, sequenceid=35, filesize=5.8 K 2024-12-09T08:19:45,695 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 8d2477d17af6e74ff8cc1d16184fd888 in 61ms, sequenceid=35, compaction requested=false; wal=null 2024-12-09T08:19:45,695 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/0000000000000000035 2024-12-09T08:19:45,696 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,696 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,697 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:19:45,698 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:45,700 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-09T08:19:45,701 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8d2477d17af6e74ff8cc1d16184fd888; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70909351, jitterRate=0.0566316694021225}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:19:45,701 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8d2477d17af6e74ff8cc1d16184fd888: Writing region info on filesystem at 1733732385598Initializing all the Stores at 1733732385598Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732385598Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732385598Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732385598Obtaining lock to block concurrent updates at 1733732385634 (+36 ms)Preparing flush snapshotting stores in 8d2477d17af6e74ff8cc1d16184fd888 at 1733732385634Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733732385634Flushing stores of testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888. at 1733732385634Flushing 8d2477d17af6e74ff8cc1d16184fd888/b: creating writer at 1733732385634Flushing 8d2477d17af6e74ff8cc1d16184fd888/b: appending metadata at 1733732385645 (+11 ms)Flushing 8d2477d17af6e74ff8cc1d16184fd888/b: closing flushed file at 1733732385645Flushing 8d2477d17af6e74ff8cc1d16184fd888/c: creating writer at 1733732385658 (+13 ms)Flushing 8d2477d17af6e74ff8cc1d16184fd888/c: appending metadata at 1733732385671 (+13 ms)Flushing 8d2477d17af6e74ff8cc1d16184fd888/c: closing flushed file at 1733732385671Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c2bd878: reopening flushed file at 1733732385682 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25b613e9: reopening flushed file at 1733732385688 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 8d2477d17af6e74ff8cc1d16184fd888 in 61ms, sequenceid=35, compaction requested=false; wal=null at 1733732385695 (+7 ms)Cleaning up temporary data from old regions at 1733732385696 (+1 ms)Region opened successfully at 1733732385701 (+5 ms) 2024-12-09T08:19:45,758 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385559, size=0 (0bytes) 2024-12-09T08:19:45,758 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385559 might be still open, length is 0 2024-12-09T08:19:45,758 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385559 2024-12-09T08:19:45,758 WARN [IPC Server handler 2 on default port 45647 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385559 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-12-09T08:19:45,758 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385559 after 0ms 2024-12-09T08:19:45,923 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:51286 [Receiving block BP-1712704723-172.17.0.2-1733732346335:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:41399:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51286 dst: /127.0.0.1:41399 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41399 remote=/127.0.0.1:51286]. Total timeout mills is 60000, 59802 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:45,923 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:49416 [Receiving block BP-1712704723-172.17.0.2-1733732346335:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:38689:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49416 dst: /127.0.0.1:38689 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:45,923 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1566349188_22 at /127.0.0.1:53670 [Receiving block BP-1712704723-172.17.0.2-1733732346335:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:45463:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53670 dst: /127.0.0.1:45463 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:45,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741930_1112 (size=5567) 2024-12-09T08:19:45,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741930_1112 (size=5567) 2024-12-09T08:19:45,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741930_1112 (size=5567) 2024-12-09T08:19:49,760 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385559 after 4002ms 2024-12-09T08:19:49,792 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:49,792 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385559 took 4035ms 2024-12-09T08:19:49,794 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385559; continuing. 2024-12-09T08:19:49,795 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385559 so closing down 2024-12-09T08:19:49,795 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T08:19:49,796 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733732385559.temp 2024-12-09T08:19:49,819 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/0000000000000000037-wal.1733732385559.temp 2024-12-09T08:19:49,819 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T08:19:49,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741933_1113 (size=5575) 2024-12-09T08:19:49,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741933_1113 (size=5575) 2024-12-09T08:19:49,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741933_1113 (size=5575) 2024-12-09T08:19:49,825 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/0000000000000000037-wal.1733732385559.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-09T08:19:49,826 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/0000000000000000037-wal.1733732385559.temp to hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/0000000000000000066 2024-12-09T08:19:49,826 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 33 ms; skipped=0; WAL=hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385559, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T08:19:49,827 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385559, journal: Splitting hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385559, size=0 (0bytes) at 1733732385758Finishing writing output for hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385559 so closing down at 1733732389795 (+4037 ms)Creating recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/0000000000000000037-wal.1733732385559.temp at 1733732389819 (+24 ms)3 split writer threads finished at 1733732389819Closed recovered edits writer path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/0000000000000000037-wal.1733732385559.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733732389825 (+6 ms)Rename recovered edits hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/0000000000000000037-wal.1733732385559.temp to hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/0000000000000000066 at 1733732389826 (+1 ms)Processed 30 edits across 1 Regions in 33 ms; skipped=0; WAL=hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385559, size=0, length=0, corrupted=false, cancelled=false at 1733732389826 2024-12-09T08:19:49,828 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385559 to hdfs://localhost:45647/hbase/oldWALs/wal.1733732385559 2024-12-09T08:19:49,829 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/0000000000000000066 2024-12-09T08:19:49,829 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T08:19:49,830 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278, archiveDir=hdfs://localhost:45647/hbase/oldWALs, maxLogs=32 2024-12-09T08:19:49,843 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732389831, exclude list is [], retry=0 2024-12-09T08:19:49,845 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41399,DS-8f003520-aa7b-4dd1-ab4c-b051e75f8195,DISK] 2024-12-09T08:19:49,845 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45463,DS-5e413aed-71fe-4f71-a7b3-1219b426abcf,DISK] 2024-12-09T08:19:49,846 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38689,DS-38bad6c6-1e8d-4290-b04e-b7bbdc65ae10,DISK] 2024-12-09T08:19:49,867 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732389831 2024-12-09T08:19:49,868 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42925:42925),(127.0.0.1/127.0.0.1:41283:41283),(127.0.0.1/127.0.0.1:42683:42683)] 2024-12-09T08:19:49,868 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T08:19:49,869 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:49,870 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d2477d17af6e74ff8cc1d16184fd888 columnFamilyName a 2024-12-09T08:19:49,870 DEBUG [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:49,874 DEBUG [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/a/92cdf36c8b5c42ffafb0902083ebd460 2024-12-09T08:19:49,874 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.HStore(327): Store=8d2477d17af6e74ff8cc1d16184fd888/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:49,874 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:49,875 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d2477d17af6e74ff8cc1d16184fd888 columnFamilyName b 2024-12-09T08:19:49,875 DEBUG [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:49,880 DEBUG [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/b/6913c5b39f21464685f62de02f345547 2024-12-09T08:19:49,880 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.HStore(327): Store=8d2477d17af6e74ff8cc1d16184fd888/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:49,880 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:49,881 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d2477d17af6e74ff8cc1d16184fd888 columnFamilyName c 2024-12-09T08:19:49,881 DEBUG [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T08:19:49,886 DEBUG [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/c/f0aa0671f657453daa54867df8bb9cf4 2024-12-09T08:19:49,886 INFO [StoreOpener-8d2477d17af6e74ff8cc1d16184fd888-1 {}] regionserver.HStore(327): Store=8d2477d17af6e74ff8cc1d16184fd888/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T08:19:49,886 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:49,887 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:49,888 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:49,889 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/0000000000000000066 2024-12-09T08:19:49,910 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T08:19:49,913 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/0000000000000000066 2024-12-09T08:19:49,913 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8d2477d17af6e74ff8cc1d16184fd888 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-09T08:19:49,925 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/.tmp/a/7131794aa0c14ee08852a8b017216f81 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733732385706/Put/seqid=0 2024-12-09T08:19:49,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741935_1115 (size=5958) 2024-12-09T08:19:49,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741935_1115 (size=5958) 2024-12-09T08:19:49,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741935_1115 (size=5958) 2024-12-09T08:19:49,931 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/.tmp/a/7131794aa0c14ee08852a8b017216f81 2024-12-09T08:19:49,947 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/.tmp/b/0ea051e297fc47419a66a8269dd3b2a7 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733732385712/Put/seqid=0 2024-12-09T08:19:49,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741936_1116 (size=5958) 2024-12-09T08:19:49,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741936_1116 (size=5958) 2024-12-09T08:19:49,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741936_1116 (size=5958) 2024-12-09T08:19:49,953 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/.tmp/b/0ea051e297fc47419a66a8269dd3b2a7 2024-12-09T08:19:49,971 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/.tmp/c/e063a462a45a46eaa547c939e67a37ee is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733732385718/Put/seqid=0 2024-12-09T08:19:49,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741937_1117 (size=5958) 2024-12-09T08:19:49,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741937_1117 (size=5958) 2024-12-09T08:19:49,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741937_1117 (size=5958) 2024-12-09T08:19:49,977 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/.tmp/c/e063a462a45a46eaa547c939e67a37ee 2024-12-09T08:19:49,981 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/.tmp/a/7131794aa0c14ee08852a8b017216f81 as hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/a/7131794aa0c14ee08852a8b017216f81 2024-12-09T08:19:49,986 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/a/7131794aa0c14ee08852a8b017216f81, entries=10, sequenceid=66, filesize=5.8 K 2024-12-09T08:19:49,987 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/.tmp/b/0ea051e297fc47419a66a8269dd3b2a7 as hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/b/0ea051e297fc47419a66a8269dd3b2a7 2024-12-09T08:19:49,991 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/b/0ea051e297fc47419a66a8269dd3b2a7, entries=10, sequenceid=66, filesize=5.8 K 2024-12-09T08:19:49,992 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/.tmp/c/e063a462a45a46eaa547c939e67a37ee as hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/c/e063a462a45a46eaa547c939e67a37ee 2024-12-09T08:19:49,997 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/c/e063a462a45a46eaa547c939e67a37ee, entries=10, sequenceid=66, filesize=5.8 K 2024-12-09T08:19:49,997 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 8d2477d17af6e74ff8cc1d16184fd888 in 84ms, sequenceid=66, compaction requested=false; wal=null 2024-12-09T08:19:49,997 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/0000000000000000066 2024-12-09T08:19:49,998 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:49,998 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:49,999 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T08:19:50,000 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8d2477d17af6e74ff8cc1d16184fd888 2024-12-09T08:19:50,001 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/hbase/data/default/testReplayEditsWrittenViaHRegion/8d2477d17af6e74ff8cc1d16184fd888/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-09T08:19:50,002 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8d2477d17af6e74ff8cc1d16184fd888; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62451483, jitterRate=-0.06940038502216339}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T08:19:50,002 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8d2477d17af6e74ff8cc1d16184fd888: Writing region info on filesystem at 1733732389868Initializing all the Stores at 1733732389869 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732389869Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732389869Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733732389869Obtaining lock to block concurrent updates at 1733732389913 (+44 ms)Preparing flush snapshotting stores in 8d2477d17af6e74ff8cc1d16184fd888 at 1733732389913Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733732389913Flushing stores of testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888. at 1733732389913Flushing 8d2477d17af6e74ff8cc1d16184fd888/a: creating writer at 1733732389913Flushing 8d2477d17af6e74ff8cc1d16184fd888/a: appending metadata at 1733732389924 (+11 ms)Flushing 8d2477d17af6e74ff8cc1d16184fd888/a: closing flushed file at 1733732389924Flushing 8d2477d17af6e74ff8cc1d16184fd888/b: creating writer at 1733732389935 (+11 ms)Flushing 8d2477d17af6e74ff8cc1d16184fd888/b: appending metadata at 1733732389947 (+12 ms)Flushing 8d2477d17af6e74ff8cc1d16184fd888/b: closing flushed file at 1733732389947Flushing 8d2477d17af6e74ff8cc1d16184fd888/c: creating writer at 1733732389957 (+10 ms)Flushing 8d2477d17af6e74ff8cc1d16184fd888/c: appending metadata at 1733732389970 (+13 ms)Flushing 8d2477d17af6e74ff8cc1d16184fd888/c: closing flushed file at 1733732389970Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42c7e1e3: reopening flushed file at 1733732389981 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44106a3d: reopening flushed file at 1733732389986 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68a5f1ec: reopening flushed file at 1733732389991 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 8d2477d17af6e74ff8cc1d16184fd888 in 84ms, sequenceid=66, compaction requested=false; wal=null at 1733732389997 (+6 ms)Cleaning up temporary data from old regions at 1733732389998 (+1 ms)Region opened successfully at 1733732390002 (+4 ms) 2024-12-09T08:19:50,013 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8d2477d17af6e74ff8cc1d16184fd888, disabling compactions & flushes 2024-12-09T08:19:50,013 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888. 2024-12-09T08:19:50,013 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888. 2024-12-09T08:19:50,013 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888. after waiting 0 ms 2024-12-09T08:19:50,013 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888. 2024-12-09T08:19:50,014 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733732385279.8d2477d17af6e74ff8cc1d16184fd888. 2024-12-09T08:19:50,014 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8d2477d17af6e74ff8cc1d16184fd888: Waiting for close lock at 1733732390012Disabling compacts and flushes for region at 1733732390012Disabling writes for close at 1733732390013 (+1 ms)Writing region close event to WAL at 1733732390014 (+1 ms)Closed at 1733732390014 2024-12-09T08:19:50,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741934_1114 (size=175) 2024-12-09T08:19:50,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741934_1114 (size=175) 2024-12-09T08:19:50,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741934_1114 (size=175) 2024-12-09T08:19:50,018 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T08:19:50,018 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733732389831) 2024-12-09T08:19:50,032 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=443 (was 438) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_835018977_22 at /127.0.0.1:49458 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_835018977_22 at /127.0.0.1:53716 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:45647 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_835018977_22 at /127.0.0.1:51324 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:45647 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1412 (was 1346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=263 (was 251) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4159 (was 4208) 2024-12-09T08:19:50,032 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1412 is superior to 1024 2024-12-09T08:19:50,032 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-09T08:19:50,033 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T08:19:50,033 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T08:19:50,033 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:19:50,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:19:50,034 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T08:19:50,034 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T08:19:50,034 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=805536183, stopped=false 2024-12-09T08:19:50,034 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=5bda946059d1,41647,1733732350732 2024-12-09T08:19:50,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34083-0x10009a83ed90002, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T08:19:50,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10009a83ed90003, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T08:19:50,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T08:19:50,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34083-0x10009a83ed90002, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:50,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10009a83ed90003, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:50,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:50,098 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T08:19:50,098 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39779-0x10009a83ed90003, quorum=127.0.0.1:55499, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T08:19:50,099 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T08:19:50,099 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34083-0x10009a83ed90002, quorum=127.0.0.1:55499, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T08:19:50,099 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T08:19:50,099 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T08:19:50,099 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:19:50,099 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5bda946059d1,34083,1733732351576' ***** 2024-12-09T08:19:50,099 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T08:19:50,099 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5bda946059d1,39779,1733732351619' ***** 2024-12-09T08:19:50,099 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T08:19:50,100 INFO [RS:2;5bda946059d1:39779 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T08:19:50,100 INFO [RS:1;5bda946059d1:34083 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T08:19:50,100 INFO [RS:2;5bda946059d1:39779 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T08:19:50,100 INFO [RS:1;5bda946059d1:34083 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T08:19:50,100 INFO [RS:1;5bda946059d1:34083 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T08:19:50,100 INFO [RS:2;5bda946059d1:39779 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T08:19:50,100 INFO [RS:1;5bda946059d1:34083 {}] regionserver.HRegionServer(3091): Received CLOSE for 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:50,100 INFO [RS:2;5bda946059d1:39779 {}] regionserver.HRegionServer(959): stopping server 5bda946059d1,39779,1733732351619 2024-12-09T08:19:50,100 INFO [RS:2;5bda946059d1:39779 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T08:19:50,100 INFO [RS:1;5bda946059d1:34083 {}] regionserver.HRegionServer(959): stopping server 5bda946059d1,34083,1733732351576 2024-12-09T08:19:50,100 INFO [RS:1;5bda946059d1:34083 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T08:19:50,100 INFO [RS:2;5bda946059d1:39779 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;5bda946059d1:39779. 2024-12-09T08:19:50,100 INFO [RS:1;5bda946059d1:34083 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;5bda946059d1:34083. 2024-12-09T08:19:50,100 DEBUG [RS:2;5bda946059d1:39779 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T08:19:50,100 DEBUG [RS:2;5bda946059d1:39779 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:19:50,100 DEBUG [RS:1;5bda946059d1:34083 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T08:19:50,100 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7a8f9beb16114c9e9ebf1936c0e75d39, disabling compactions & flushes 2024-12-09T08:19:50,100 DEBUG [RS:1;5bda946059d1:34083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:19:50,100 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:50,100 INFO [RS:2;5bda946059d1:39779 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T08:19:50,100 INFO [RS:2;5bda946059d1:39779 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T08:19:50,100 INFO [RS:1;5bda946059d1:34083 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T08:19:50,100 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:50,100 INFO [RS:2;5bda946059d1:39779 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T08:19:50,100 DEBUG [RS:1;5bda946059d1:34083 {}] regionserver.HRegionServer(1325): Online Regions={7a8f9beb16114c9e9ebf1936c0e75d39=testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39.} 2024-12-09T08:19:50,100 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. after waiting 0 ms 2024-12-09T08:19:50,100 DEBUG [RS:1;5bda946059d1:34083 {}] regionserver.HRegionServer(1351): Waiting on 7a8f9beb16114c9e9ebf1936c0e75d39 2024-12-09T08:19:50,100 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:50,100 INFO [RS:2;5bda946059d1:39779 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T08:19:50,100 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T08:19:50,101 INFO [RS:2;5bda946059d1:39779 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T08:19:50,101 DEBUG [RS:2;5bda946059d1:39779 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-09T08:19:50,101 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T08:19:50,101 DEBUG [RS:2;5bda946059d1:39779 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T08:19:50,101 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T08:19:50,101 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T08:19:50,101 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T08:19:50,101 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T08:19:50,101 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T08:19:50,101 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-12-09T08:19:50,105 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7a8f9beb16114c9e9ebf1936c0e75d39/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-09T08:19:50,105 INFO [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:50,105 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7a8f9beb16114c9e9ebf1936c0e75d39: Waiting for close lock at 1733732390100Running coprocessor pre-close hooks at 1733732390100Disabling compacts and flushes for region at 1733732390100Disabling writes for close at 1733732390100Writing region close event to WAL at 1733732390101 (+1 ms)Running coprocessor post-close hooks at 1733732390105 (+4 ms)Closed at 1733732390105 2024-12-09T08:19:50,106 DEBUG [RS_CLOSE_REGION-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39. 2024-12-09T08:19:50,127 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/hbase/meta/1588230740/.tmp/info/7a89889bb1114ec590d6bc96f41d18fa is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733732366608.7a8f9beb16114c9e9ebf1936c0e75d39./info:regioninfo/1733732369725/Put/seqid=0 2024-12-09T08:19:50,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741938_1118 (size=8243) 2024-12-09T08:19:50,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741938_1118 (size=8243) 2024-12-09T08:19:50,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741938_1118 (size=8243) 2024-12-09T08:19:50,134 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/hbase/meta/1588230740/.tmp/info/7a89889bb1114ec590d6bc96f41d18fa 2024-12-09T08:19:50,151 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/hbase/meta/1588230740/.tmp/ns/4cbe6d6c193a47698c5ceb77a8beb154 is 43, key is default/ns:d/1733732353956/Put/seqid=0 2024-12-09T08:19:50,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741939_1119 (size=5153) 2024-12-09T08:19:50,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741939_1119 (size=5153) 2024-12-09T08:19:50,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741939_1119 (size=5153) 2024-12-09T08:19:50,158 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/hbase/meta/1588230740/.tmp/ns/4cbe6d6c193a47698c5ceb77a8beb154 2024-12-09T08:19:50,159 INFO [regionserver/5bda946059d1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T08:19:50,161 INFO [regionserver/5bda946059d1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T08:19:50,178 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/hbase/meta/1588230740/.tmp/table/4bae37ab5dba4c75a65bc37e59ee5742 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733732367026/Put/seqid=0 2024-12-09T08:19:50,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741940_1120 (size=5431) 2024-12-09T08:19:50,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741940_1120 (size=5431) 2024-12-09T08:19:50,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741940_1120 (size=5431) 2024-12-09T08:19:50,185 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/hbase/meta/1588230740/.tmp/table/4bae37ab5dba4c75a65bc37e59ee5742 2024-12-09T08:19:50,192 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/hbase/meta/1588230740/.tmp/info/7a89889bb1114ec590d6bc96f41d18fa as hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/hbase/meta/1588230740/info/7a89889bb1114ec590d6bc96f41d18fa 2024-12-09T08:19:50,197 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/hbase/meta/1588230740/info/7a89889bb1114ec590d6bc96f41d18fa, entries=18, sequenceid=21, filesize=8.0 K 2024-12-09T08:19:50,198 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/hbase/meta/1588230740/.tmp/ns/4cbe6d6c193a47698c5ceb77a8beb154 as hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/hbase/meta/1588230740/ns/4cbe6d6c193a47698c5ceb77a8beb154 2024-12-09T08:19:50,205 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/hbase/meta/1588230740/ns/4cbe6d6c193a47698c5ceb77a8beb154, entries=2, sequenceid=21, filesize=5.0 K 2024-12-09T08:19:50,206 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/hbase/meta/1588230740/.tmp/table/4bae37ab5dba4c75a65bc37e59ee5742 as hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/hbase/meta/1588230740/table/4bae37ab5dba4c75a65bc37e59ee5742 2024-12-09T08:19:50,211 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/hbase/meta/1588230740/table/4bae37ab5dba4c75a65bc37e59ee5742, entries=2, sequenceid=21, filesize=5.3 K 2024-12-09T08:19:50,212 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 111ms, sequenceid=21, compaction requested=false 2024-12-09T08:19:50,217 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-12-09T08:19:50,217 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T08:19:50,217 INFO [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T08:19:50,217 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733732390101Running coprocessor pre-close hooks at 1733732390101Disabling compacts and flushes for region at 1733732390101Disabling writes for close at 1733732390101Obtaining lock to block concurrent updates at 1733732390101Preparing flush snapshotting stores in 1588230740 at 1733732390101Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1733732390102 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733732390102Flushing 1588230740/info: creating writer at 1733732390102Flushing 1588230740/info: appending metadata at 1733732390126 (+24 ms)Flushing 1588230740/info: closing flushed file at 1733732390126Flushing 1588230740/ns: creating writer at 1733732390139 (+13 ms)Flushing 1588230740/ns: appending metadata at 1733732390151 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1733732390151Flushing 1588230740/table: creating writer at 1733732390165 (+14 ms)Flushing 1588230740/table: appending metadata at 1733732390178 (+13 ms)Flushing 1588230740/table: closing flushed file at 1733732390178Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48c819da: reopening flushed file at 1733732390191 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@272fb16: reopening flushed file at 1733732390197 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f9883b7: reopening flushed file at 1733732390205 (+8 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 111ms, sequenceid=21, compaction requested=false at 1733732390212 (+7 ms)Writing region close event to WAL at 1733732390213 (+1 ms)Running coprocessor post-close hooks at 1733732390217 (+4 ms)Closed at 1733732390217 2024-12-09T08:19:50,218 DEBUG [RS_CLOSE_META-regionserver/5bda946059d1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T08:19:50,301 INFO [RS:1;5bda946059d1:34083 {}] regionserver.HRegionServer(976): stopping server 5bda946059d1,34083,1733732351576; all regions closed. 2024-12-09T08:19:50,301 INFO [RS:2;5bda946059d1:39779 {}] regionserver.HRegionServer(976): stopping server 5bda946059d1,39779,1733732351619; all regions closed. 2024-12-09T08:19:50,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741835_1011 (size=3621) 2024-12-09T08:19:50,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741835_1011 (size=3621) 2024-12-09T08:19:50,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741836_1012 (size=9487) 2024-12-09T08:19:50,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741835_1011 (size=3621) 2024-12-09T08:19:50,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741836_1012 (size=9487) 2024-12-09T08:19:50,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741836_1012 (size=9487) 2024-12-09T08:19:50,306 DEBUG [RS:1;5bda946059d1:34083 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/oldWALs 2024-12-09T08:19:50,307 INFO [RS:1;5bda946059d1:34083 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5bda946059d1%2C34083%2C1733732351576:(num 1733732353180) 2024-12-09T08:19:50,307 DEBUG [RS:1;5bda946059d1:34083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:19:50,307 INFO [RS:1;5bda946059d1:34083 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T08:19:50,307 INFO [RS:1;5bda946059d1:34083 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T08:19:50,307 INFO [RS:1;5bda946059d1:34083 {}] hbase.ChoreService(370): Chore service for: regionserver/5bda946059d1:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-09T08:19:50,307 DEBUG [RS:2;5bda946059d1:39779 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/oldWALs 2024-12-09T08:19:50,307 INFO [RS:1;5bda946059d1:34083 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T08:19:50,307 INFO [RS:2;5bda946059d1:39779 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5bda946059d1%2C39779%2C1733732351619.meta:.meta(num 1733732353537) 2024-12-09T08:19:50,307 INFO [RS:1;5bda946059d1:34083 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T08:19:50,307 INFO [RS:1;5bda946059d1:34083 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T08:19:50,307 INFO [RS:1;5bda946059d1:34083 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T08:19:50,307 INFO [regionserver/5bda946059d1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T08:19:50,308 INFO [RS:1;5bda946059d1:34083 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34083 2024-12-09T08:19:50,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741834_1010 (size=175) 2024-12-09T08:19:50,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741834_1010 (size=175) 2024-12-09T08:19:50,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741834_1010 (size=175) 2024-12-09T08:19:50,314 DEBUG [RS:2;5bda946059d1:39779 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/oldWALs 2024-12-09T08:19:50,314 INFO [RS:2;5bda946059d1:39779 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5bda946059d1%2C39779%2C1733732351619:(num 1733732353180) 2024-12-09T08:19:50,314 DEBUG [RS:2;5bda946059d1:39779 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T08:19:50,314 INFO [RS:2;5bda946059d1:39779 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T08:19:50,314 INFO [RS:2;5bda946059d1:39779 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T08:19:50,314 INFO [RS:2;5bda946059d1:39779 {}] hbase.ChoreService(370): Chore service for: regionserver/5bda946059d1:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T08:19:50,315 INFO [RS:2;5bda946059d1:39779 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T08:19:50,315 INFO [regionserver/5bda946059d1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T08:19:50,315 INFO [RS:2;5bda946059d1:39779 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39779 2024-12-09T08:19:50,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34083-0x10009a83ed90002, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/5bda946059d1,34083,1733732351576 2024-12-09T08:19:50,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T08:19:50,319 INFO [RS:1;5bda946059d1:34083 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T08:19:50,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10009a83ed90003, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/5bda946059d1,39779,1733732351619 2024-12-09T08:19:50,329 INFO [RS:2;5bda946059d1:39779 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T08:19:50,340 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5bda946059d1,34083,1733732351576] 2024-12-09T08:19:50,361 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/5bda946059d1,34083,1733732351576 already deleted, retry=false 2024-12-09T08:19:50,361 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 5bda946059d1,34083,1733732351576 expired; onlineServers=1 2024-12-09T08:19:50,361 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5bda946059d1,39779,1733732351619] 2024-12-09T08:19:50,371 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/5bda946059d1,39779,1733732351619 already deleted, retry=false 2024-12-09T08:19:50,371 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 5bda946059d1,39779,1733732351619 expired; onlineServers=0 2024-12-09T08:19:50,372 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '5bda946059d1,41647,1733732350732' ***** 2024-12-09T08:19:50,372 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-09T08:19:50,372 INFO [M:0;5bda946059d1:41647 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T08:19:50,372 INFO [M:0;5bda946059d1:41647 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T08:19:50,372 DEBUG [M:0;5bda946059d1:41647 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T08:19:50,372 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T08:19:50,372 DEBUG [M:0;5bda946059d1:41647 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T08:19:50,372 DEBUG [master/5bda946059d1:0:becomeActiveMaster-HFileCleaner.small.0-1733732352750 {}] cleaner.HFileCleaner(306): Exit Thread[master/5bda946059d1:0:becomeActiveMaster-HFileCleaner.small.0-1733732352750,5,FailOnTimeoutGroup] 2024-12-09T08:19:50,372 DEBUG [master/5bda946059d1:0:becomeActiveMaster-HFileCleaner.large.0-1733732352749 {}] cleaner.HFileCleaner(306): Exit Thread[master/5bda946059d1:0:becomeActiveMaster-HFileCleaner.large.0-1733732352749,5,FailOnTimeoutGroup] 2024-12-09T08:19:50,372 INFO [M:0;5bda946059d1:41647 {}] hbase.ChoreService(370): Chore service for: master/5bda946059d1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-09T08:19:50,372 INFO [M:0;5bda946059d1:41647 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T08:19:50,372 DEBUG [M:0;5bda946059d1:41647 {}] master.HMaster(1795): Stopping service threads 2024-12-09T08:19:50,373 INFO [M:0;5bda946059d1:41647 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T08:19:50,373 INFO [M:0;5bda946059d1:41647 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T08:19:50,373 INFO [M:0;5bda946059d1:41647 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T08:19:50,373 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T08:19:50,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T08:19:50,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T08:19:50,382 DEBUG [M:0;5bda946059d1:41647 {}] zookeeper.ZKUtil(347): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-09T08:19:50,382 WARN [M:0;5bda946059d1:41647 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-09T08:19:50,383 INFO [M:0;5bda946059d1:41647 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/.lastflushedseqids 2024-12-09T08:19:50,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741941_1121 (size=138) 2024-12-09T08:19:50,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741941_1121 (size=138) 2024-12-09T08:19:50,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741941_1121 (size=138) 2024-12-09T08:19:50,396 INFO [M:0;5bda946059d1:41647 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T08:19:50,397 INFO [M:0;5bda946059d1:41647 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T08:19:50,397 DEBUG [M:0;5bda946059d1:41647 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T08:19:50,397 INFO [M:0;5bda946059d1:41647 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T08:19:50,397 DEBUG [M:0;5bda946059d1:41647 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T08:19:50,397 DEBUG [M:0;5bda946059d1:41647 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T08:19:50,397 DEBUG [M:0;5bda946059d1:41647 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T08:19:50,397 INFO [M:0;5bda946059d1:41647 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.32 KB heapSize=83.72 KB 2024-12-09T08:19:50,410 DEBUG [M:0;5bda946059d1:41647 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c1c47531220446f39691579bf0feb5fe is 82, key is hbase:meta,,1/info:regioninfo/1733732353638/Put/seqid=0 2024-12-09T08:19:50,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741942_1122 (size=5672) 2024-12-09T08:19:50,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741942_1122 (size=5672) 2024-12-09T08:19:50,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741942_1122 (size=5672) 2024-12-09T08:19:50,416 INFO [M:0;5bda946059d1:41647 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c1c47531220446f39691579bf0feb5fe 2024-12-09T08:19:50,435 DEBUG [M:0;5bda946059d1:41647 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f3b31542e4384905a18f6af9b8c2a836 is 1074, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733732367033/Put/seqid=0 2024-12-09T08:19:50,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34083-0x10009a83ed90002, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T08:19:50,440 INFO [RS:1;5bda946059d1:34083 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T08:19:50,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34083-0x10009a83ed90002, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T08:19:50,440 INFO [RS:1;5bda946059d1:34083 {}] regionserver.HRegionServer(1031): Exiting; stopping=5bda946059d1,34083,1733732351576; zookeeper connection closed. 2024-12-09T08:19:50,440 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@29b13760 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@29b13760 2024-12-09T08:19:50,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741943_1123 (size=7753) 2024-12-09T08:19:50,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741943_1123 (size=7753) 2024-12-09T08:19:50,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741943_1123 (size=7753) 2024-12-09T08:19:50,442 INFO [M:0;5bda946059d1:41647 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.58 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f3b31542e4384905a18f6af9b8c2a836 2024-12-09T08:19:50,447 INFO [M:0;5bda946059d1:41647 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f3b31542e4384905a18f6af9b8c2a836 2024-12-09T08:19:50,450 INFO [RS:2;5bda946059d1:39779 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T08:19:50,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10009a83ed90003, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T08:19:50,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39779-0x10009a83ed90003, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T08:19:50,450 INFO [RS:2;5bda946059d1:39779 {}] regionserver.HRegionServer(1031): Exiting; stopping=5bda946059d1,39779,1733732351619; zookeeper connection closed. 2024-12-09T08:19:50,451 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@8352d6c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@8352d6c 2024-12-09T08:19:50,451 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-09T08:19:50,464 DEBUG [M:0;5bda946059d1:41647 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dab44084422a46e3b90116d66a042e47 is 69, key is 5bda946059d1,34083,1733732351576/rs:state/1733732352912/Put/seqid=0 2024-12-09T08:19:50,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741944_1124 (size=5440) 2024-12-09T08:19:50,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741944_1124 (size=5440) 2024-12-09T08:19:50,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741944_1124 (size=5440) 2024-12-09T08:19:50,470 INFO [M:0;5bda946059d1:41647 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dab44084422a46e3b90116d66a042e47 2024-12-09T08:19:50,474 INFO [M:0;5bda946059d1:41647 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dab44084422a46e3b90116d66a042e47 2024-12-09T08:19:50,476 DEBUG [M:0;5bda946059d1:41647 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c1c47531220446f39691579bf0feb5fe as hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c1c47531220446f39691579bf0feb5fe 2024-12-09T08:19:50,481 INFO [M:0;5bda946059d1:41647 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c1c47531220446f39691579bf0feb5fe, entries=8, sequenceid=168, filesize=5.5 K 2024-12-09T08:19:50,482 DEBUG [M:0;5bda946059d1:41647 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f3b31542e4384905a18f6af9b8c2a836 as hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f3b31542e4384905a18f6af9b8c2a836 2024-12-09T08:19:50,487 INFO [M:0;5bda946059d1:41647 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f3b31542e4384905a18f6af9b8c2a836 2024-12-09T08:19:50,487 INFO [M:0;5bda946059d1:41647 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f3b31542e4384905a18f6af9b8c2a836, entries=17, sequenceid=168, filesize=7.6 K 2024-12-09T08:19:50,488 DEBUG [M:0;5bda946059d1:41647 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dab44084422a46e3b90116d66a042e47 as hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/dab44084422a46e3b90116d66a042e47 2024-12-09T08:19:50,492 INFO [M:0;5bda946059d1:41647 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dab44084422a46e3b90116d66a042e47 2024-12-09T08:19:50,493 INFO [M:0;5bda946059d1:41647 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45647/user/jenkins/test-data/ad04e9ae-191f-f8ba-973e-5ca9bf36e3b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/dab44084422a46e3b90116d66a042e47, entries=3, sequenceid=168, filesize=5.3 K 2024-12-09T08:19:50,494 INFO [M:0;5bda946059d1:41647 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.32 KB/69960, heapSize ~83.42 KB/85424, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 97ms, sequenceid=168, compaction requested=false 2024-12-09T08:19:50,495 INFO [M:0;5bda946059d1:41647 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T08:19:50,495 DEBUG [M:0;5bda946059d1:41647 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733732390397Disabling compacts and flushes for region at 1733732390397Disabling writes for close at 1733732390397Obtaining lock to block concurrent updates at 1733732390397Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733732390397Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69960, getHeapSize=85664, getOffHeapSize=0, getCellsCount=195 at 1733732390397Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733732390398 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733732390398Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733732390410 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733732390410Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733732390420 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733732390435 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733732390435Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733732390448 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733732390463 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733732390463Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@227c274f: reopening flushed file at 1733732390475 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@776502e1: reopening flushed file at 1733732390481 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b03511d: reopening flushed file at 1733732390487 (+6 ms)Finished flush of dataSize ~68.32 KB/69960, heapSize ~83.42 KB/85424, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 97ms, sequenceid=168, compaction requested=false at 1733732390494 (+7 ms)Writing region close event to WAL at 1733732390495 (+1 ms)Closed at 1733732390495 2024-12-09T08:19:50,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41399 is added to blk_1073741830_1006 (size=83582) 2024-12-09T08:19:50,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38689 is added to blk_1073741830_1006 (size=83582) 2024-12-09T08:19:50,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45463 is added to blk_1073741830_1006 (size=83582) 2024-12-09T08:19:50,498 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T08:19:50,498 INFO [M:0;5bda946059d1:41647 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-09T08:19:50,498 INFO [M:0;5bda946059d1:41647 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41647 2024-12-09T08:19:50,498 INFO [M:0;5bda946059d1:41647 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T08:19:50,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T08:19:50,608 INFO [M:0;5bda946059d1:41647 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T08:19:50,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x10009a83ed90000, quorum=127.0.0.1:55499, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T08:19:50,617 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732370015/wal.1733732370374 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732370015/wal.1733732370374 (inode 16655) Holder DFSClient_NONMAPREDUCE_1566349188_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733732370015/wal.1733732370374 (inode 16655) Holder DFSClient_NONMAPREDUCE_1566349188_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-09T08:19:50,619 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733732362114/wal.1733732362226 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:50,619 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733732385278/wal.1733732385559 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:50,622 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732370550/wal.1733732379875 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732370550/wal.1733732379875 (inode 16678) Holder DFSClient_NONMAPREDUCE_1566349188_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733732370550/wal.1733732379875 (inode 16678) Holder DFSClient_NONMAPREDUCE_1566349188_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-09T08:19:50,622 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733732355427/wal.1733732355513 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:50,624 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733732380098/wal.1733732380618 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733732380098/wal.1733732380618 (inode 16704) Holder DFSClient_NONMAPREDUCE_1566349188_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733732380098/wal.1733732380618 (inode 16704) Holder DFSClient_NONMAPREDUCE_1566349188_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-09T08:19:50,624 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733732380791/wal.1733732380857 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T08:19:50,626 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884/wal.1733732355245 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884/wal.1733732355245 (inode 16485) Holder DFSClient_NONMAPREDUCE_1566349188_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733732354884/wal.1733732355245 (inode 16485) Holder DFSClient_NONMAPREDUCE_1566349188_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-09T08:19:50,629 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235/wal.1733732354625 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235/wal.1733732354625 (inode 16462) Holder DFSClient_NONMAPREDUCE_1566349188_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733732354235/wal.1733732354625 (inode 16462) Holder DFSClient_NONMAPREDUCE_1566349188_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-09T08:19:50,658 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@420bc738{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T08:19:50,661 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T08:19:50,661 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T08:19:50,661 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T08:19:50,661 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/hadoop.log.dir/,STOPPED} 2024-12-09T08:19:50,664 WARN [BP-1712704723-172.17.0.2-1733732346335 heartbeating to localhost/127.0.0.1:45647 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T08:19:50,664 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T08:19:50,665 WARN [BP-1712704723-172.17.0.2-1733732346335 heartbeating to localhost/127.0.0.1:45647 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1712704723-172.17.0.2-1733732346335 (Datanode Uuid d6a66b69-c727-452f-a95a-1253d0f4e848) service to localhost/127.0.0.1:45647 2024-12-09T08:19:50,665 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T08:19:50,666 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/cluster_95cc11f3-affe-6e07-bfcd-38f5721bf4f5/data/data5/current/BP-1712704723-172.17.0.2-1733732346335 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T08:19:50,667 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/cluster_95cc11f3-affe-6e07-bfcd-38f5721bf4f5/data/data6/current/BP-1712704723-172.17.0.2-1733732346335 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T08:19:50,667 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T08:19:50,669 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64052af6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T08:19:50,669 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T08:19:50,670 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T08:19:50,670 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T08:19:50,670 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/hadoop.log.dir/,STOPPED} 2024-12-09T08:19:50,671 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T08:19:50,671 WARN [BP-1712704723-172.17.0.2-1733732346335 heartbeating to localhost/127.0.0.1:45647 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T08:19:50,671 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T08:19:50,671 WARN [BP-1712704723-172.17.0.2-1733732346335 heartbeating to localhost/127.0.0.1:45647 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1712704723-172.17.0.2-1733732346335 (Datanode Uuid 40980e3f-e063-4027-a6f2-2240b9222764) service to localhost/127.0.0.1:45647 2024-12-09T08:19:50,672 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/cluster_95cc11f3-affe-6e07-bfcd-38f5721bf4f5/data/data3/current/BP-1712704723-172.17.0.2-1733732346335 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T08:19:50,672 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/cluster_95cc11f3-affe-6e07-bfcd-38f5721bf4f5/data/data4/current/BP-1712704723-172.17.0.2-1733732346335 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T08:19:50,672 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T08:19:50,674 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@501095cc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T08:19:50,674 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T08:19:50,674 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T08:19:50,675 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T08:19:50,675 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/hadoop.log.dir/,STOPPED} 2024-12-09T08:19:50,676 WARN [BP-1712704723-172.17.0.2-1733732346335 heartbeating to localhost/127.0.0.1:45647 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T08:19:50,676 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T08:19:50,676 WARN [BP-1712704723-172.17.0.2-1733732346335 heartbeating to localhost/127.0.0.1:45647 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1712704723-172.17.0.2-1733732346335 (Datanode Uuid 8817ac4d-f808-4e08-9488-ec5e11a1e412) service to localhost/127.0.0.1:45647 2024-12-09T08:19:50,676 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T08:19:50,676 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/cluster_95cc11f3-affe-6e07-bfcd-38f5721bf4f5/data/data1/current/BP-1712704723-172.17.0.2-1733732346335 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T08:19:50,676 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/cluster_95cc11f3-affe-6e07-bfcd-38f5721bf4f5/data/data2/current/BP-1712704723-172.17.0.2-1733732346335 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T08:19:50,677 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T08:19:50,682 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c03e044{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T08:19:50,683 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T08:19:50,683 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T08:19:50,683 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T08:19:50,683 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f33a0023-836d-39e6-9c36-fb36f4d2acbd/hadoop.log.dir/,STOPPED} 2024-12-09T08:19:50,691 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-09T08:19:50,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down